From 1eff320d5e7fb5510d13016e0826b14e5cf7f686 Mon Sep 17 00:00:00 2001 From: Kenneth Knowles Date: Mon, 24 Oct 2016 12:57:37 -0700 Subject: [PATCH 1/2] Remove pieces of Trigger now owned by TriggerStateMachine --- .../sdk/transforms/windowing/AfterAll.java | 49 -- .../windowing/AfterDelayFromFirstElement.java | 99 ---- .../sdk/transforms/windowing/AfterEach.java | 61 --- .../sdk/transforms/windowing/AfterFirst.java | 50 -- .../sdk/transforms/windowing/AfterPane.java | 52 -- .../windowing/AfterProcessingTime.java | 7 - .../AfterSynchronizedProcessingTime.java | 7 - .../transforms/windowing/AfterWatermark.java | 158 ------ .../transforms/windowing/DefaultTrigger.java | 35 -- .../beam/sdk/transforms/windowing/Never.java | 17 - .../windowing/OrFinallyTrigger.java | 46 -- .../sdk/transforms/windowing/Repeatedly.java | 30 -- .../sdk/transforms/windowing/Trigger.java | 412 ++------------ .../beam/sdk/util/ExecutableTrigger.java | 40 +- .../beam/sdk/util/FinishedTriggers.java | 44 -- .../beam/sdk/util/FinishedTriggersBitSet.java | 67 --- .../beam/sdk/util/FinishedTriggersSet.java | 72 --- .../beam/sdk/util/ReshuffleTrigger.java | 14 - .../beam/sdk/util/TriggerContextFactory.java | 507 ------------------ .../transforms/windowing/AfterAllTest.java | 98 ---- .../transforms/windowing/AfterEachTest.java | 64 --- .../transforms/windowing/AfterFirstTest.java | 120 ----- .../transforms/windowing/AfterPaneTest.java | 77 --- .../windowing/AfterProcessingTimeTest.java | 94 ---- .../AfterSynchronizedProcessingTimeTest.java | 75 --- .../windowing/AfterWatermarkTest.java | 308 ----------- .../windowing/DefaultTriggerTest.java | 130 ----- .../sdk/transforms/windowing/NeverTest.java | 34 +- .../windowing/OrFinallyTriggerTest.java | 136 ----- .../transforms/windowing/RepeatedlyTest.java | 161 +----- .../sdk/transforms/windowing/StubTrigger.java | 17 - .../sdk/transforms/windowing/TriggerTest.java | 28 - .../beam/sdk/util/ExecutableTriggerTest.java | 18 - .../sdk/util/FinishedTriggersBitSetTest.java | 55 -- .../sdk/util/FinishedTriggersProperties.java | 110 ---- .../sdk/util/FinishedTriggersSetTest.java | 60 --- .../beam/sdk/util/ReshuffleTriggerTest.java | 23 - .../apache/beam/sdk/util/TriggerTester.java | 410 -------------- 38 files changed, 77 insertions(+), 3708 deletions(-) delete mode 100644 sdks/java/core/src/main/java/org/apache/beam/sdk/util/FinishedTriggers.java delete mode 100644 sdks/java/core/src/main/java/org/apache/beam/sdk/util/FinishedTriggersBitSet.java delete mode 100644 sdks/java/core/src/main/java/org/apache/beam/sdk/util/FinishedTriggersSet.java delete mode 100644 sdks/java/core/src/main/java/org/apache/beam/sdk/util/TriggerContextFactory.java delete mode 100644 sdks/java/core/src/test/java/org/apache/beam/sdk/util/FinishedTriggersBitSetTest.java delete mode 100644 sdks/java/core/src/test/java/org/apache/beam/sdk/util/FinishedTriggersProperties.java delete mode 100644 sdks/java/core/src/test/java/org/apache/beam/sdk/util/FinishedTriggersSetTest.java delete mode 100644 sdks/java/core/src/test/java/org/apache/beam/sdk/util/TriggerTester.java diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/AfterAll.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/AfterAll.java index 0e37d332d248..c3f084893a4c 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/AfterAll.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/AfterAll.java @@ -24,7 +24,6 @@ import java.util.List; import org.apache.beam.sdk.annotations.Experimental; import org.apache.beam.sdk.transforms.windowing.Trigger.OnceTrigger; -import org.apache.beam.sdk.util.ExecutableTrigger; import org.joda.time.Instant; /** @@ -45,27 +44,6 @@ public static AfterAll of(OnceTrigger... triggers) { return new AfterAll(Arrays.asList(triggers)); } - @Override - public void onElement(OnElementContext c) throws Exception { - for (ExecutableTrigger subTrigger : c.trigger().unfinishedSubTriggers()) { - // Since subTriggers are all OnceTriggers, they must either CONTINUE or FIRE_AND_FINISH. - // invokeElement will automatically mark the finish bit if they return FIRE_AND_FINISH. - subTrigger.invokeOnElement(c); - } - } - - @Override - public void onMerge(OnMergeContext c) throws Exception { - for (ExecutableTrigger subTrigger : c.trigger().subTriggers()) { - subTrigger.invokeOnMerge(c); - } - boolean allFinished = true; - for (ExecutableTrigger subTrigger1 : c.trigger().subTriggers()) { - allFinished &= c.forTrigger(subTrigger1).trigger().isFinished(); - } - c.trigger().setFinished(allFinished); - } - @Override public Instant getWatermarkThatGuaranteesFiring(BoundedWindow window) { // This trigger will fire after the latest of its sub-triggers. @@ -84,33 +62,6 @@ public OnceTrigger getContinuationTrigger(List continuationTriggers) { return new AfterAll(continuationTriggers); } - /** - * {@inheritDoc} - * - * @return {@code true} if all subtriggers return {@code true}. - */ - @Override - public boolean shouldFire(TriggerContext context) throws Exception { - for (ExecutableTrigger subtrigger : context.trigger().subTriggers()) { - if (!context.forTrigger(subtrigger).trigger().isFinished() - && !subtrigger.invokeShouldFire(context)) { - return false; - } - } - return true; - } - - /** - * Invokes {@link #onFire} for all subtriggers, eliding redundant calls to {@link #shouldFire} - * because they all must be ready to fire. - */ - @Override - public void onOnlyFiring(TriggerContext context) throws Exception { - for (ExecutableTrigger subtrigger : context.trigger().subTriggers()) { - subtrigger.invokeOnFire(context); - } - } - @Override public String toString() { StringBuilder builder = new StringBuilder("AfterAll.of("); diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/AfterDelayFromFirstElement.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/AfterDelayFromFirstElement.java index 6078b346d498..9daecb20379a 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/AfterDelayFromFirstElement.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/AfterDelayFromFirstElement.java @@ -18,11 +18,9 @@ package org.apache.beam.sdk.transforms.windowing; import com.google.common.collect.ImmutableList; -import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import java.util.List; import java.util.Locale; import java.util.Objects; -import javax.annotation.Nullable; import org.apache.beam.sdk.annotations.Experimental; import org.apache.beam.sdk.coders.InstantCoder; import org.apache.beam.sdk.transforms.Combine; @@ -31,10 +29,6 @@ import org.apache.beam.sdk.transforms.windowing.Trigger.OnceTrigger; import org.apache.beam.sdk.util.TimeDomain; import org.apache.beam.sdk.util.state.AccumulatorCombiningState; -import org.apache.beam.sdk.util.state.CombiningState; -import org.apache.beam.sdk.util.state.MergingStateAccessor; -import org.apache.beam.sdk.util.state.StateAccessor; -import org.apache.beam.sdk.util.state.StateMerging; import org.apache.beam.sdk.util.state.StateTag; import org.apache.beam.sdk.util.state.StateTags; import org.joda.time.Duration; @@ -61,12 +55,6 @@ public abstract class AfterDelayFromFirstElement extends OnceTrigger { private static final PeriodFormatter PERIOD_FORMATTER = PeriodFormat.wordBased(Locale.ENGLISH); - /** - * To complete an implementation, return the desired time from the TriggerContext. - */ - @Nullable - public abstract Instant getCurrentTime(Trigger.TriggerContext context); - /** * To complete an implementation, return a new instance like this one, but incorporating * the provided timestamp mapping functions. Generally should be used by calling the @@ -92,10 +80,6 @@ public AfterDelayFromFirstElement( this.timeDomain = timeDomain; } - private Instant getTargetTimestamp(OnElementContext c) { - return computeTargetTimestamp(c.currentProcessingTime()); - } - /** * The time domain according for which this trigger sets timers. */ @@ -169,94 +153,11 @@ private AfterDelayFromFirstElement newWith( .build()); } - @Override - @SuppressFBWarnings(value = "RV_RETURN_VALUE_IGNORED_NO_SIDE_EFFECT", justification = - "prefetch side effect") - public void prefetchOnElement(StateAccessor state) { - state.access(DELAYED_UNTIL_TAG).readLater(); - } - - @Override - public void onElement(OnElementContext c) throws Exception { - CombiningState delayUntilState = c.state().access(DELAYED_UNTIL_TAG); - Instant oldDelayUntil = delayUntilState.read(); - - // Since processing time can only advance, resulting in target wake-up times we would - // ignore anyhow, we don't bother with it if it is already set. - if (oldDelayUntil != null) { - return; - } - - Instant targetTimestamp = getTargetTimestamp(c); - delayUntilState.add(targetTimestamp); - c.setTimer(targetTimestamp, timeDomain); - } - - @Override - public void prefetchOnMerge(MergingStateAccessor state) { - super.prefetchOnMerge(state); - StateMerging.prefetchCombiningValues(state, DELAYED_UNTIL_TAG); - } - - @Override - public void onMerge(OnMergeContext c) throws Exception { - // NOTE: We could try to delete all timers which are still active, but we would - // need access to a timer context for each merging window. - // for (CombiningValueStateInternal, Instant> state : - // c.state().accessInEachMergingWindow(DELAYED_UNTIL_TAG).values()) { - // Instant timestamp = state.get().read(); - // if (timestamp != null) { - // .deleteTimer(timestamp, timeDomain); - // } - // } - // Instead let them fire and be ignored. - - // If the trigger is already finished, there is no way it will become re-activated - if (c.trigger().isFinished()) { - StateMerging.clear(c.state(), DELAYED_UNTIL_TAG); - // NOTE: We do not attempt to delete the timers. - return; - } - - // Determine the earliest point across all the windows, and delay to that. - StateMerging.mergeCombiningValues(c.state(), DELAYED_UNTIL_TAG); - - Instant earliestTargetTime = c.state().access(DELAYED_UNTIL_TAG).read(); - if (earliestTargetTime != null) { - c.setTimer(earliestTargetTime, timeDomain); - } - } - - @Override - @SuppressFBWarnings(value = "RV_RETURN_VALUE_IGNORED_NO_SIDE_EFFECT", justification = - "prefetch side effect") - public void prefetchShouldFire(StateAccessor state) { - state.access(DELAYED_UNTIL_TAG).readLater(); - } - - @Override - public void clear(TriggerContext c) throws Exception { - c.state().access(DELAYED_UNTIL_TAG).clear(); - } - @Override public Instant getWatermarkThatGuaranteesFiring(BoundedWindow window) { return BoundedWindow.TIMESTAMP_MAX_VALUE; } - @Override - public boolean shouldFire(Trigger.TriggerContext context) throws Exception { - Instant delayedUntil = context.state().access(DELAYED_UNTIL_TAG).read(); - return delayedUntil != null - && getCurrentTime(context) != null - && getCurrentTime(context).isAfter(delayedUntil); - } - - @Override - protected void onOnlyFiring(Trigger.TriggerContext context) throws Exception { - clear(context); - } - protected Instant computeTargetTimestamp(Instant time) { Instant result = time; for (SerializableFunction timestampMapper : timestampMappers) { diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/AfterEach.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/AfterEach.java index 961d97f432fd..872ad46e16d5 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/AfterEach.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/AfterEach.java @@ -23,7 +23,6 @@ import java.util.Arrays; import java.util.List; import org.apache.beam.sdk.annotations.Experimental; -import org.apache.beam.sdk.util.ExecutableTrigger; import org.joda.time.Instant; /** @@ -58,41 +57,6 @@ public static AfterEach inOrder(Trigger... triggers) { return new AfterEach(Arrays.asList(triggers)); } - @Override - public void onElement(OnElementContext c) throws Exception { - if (!c.trigger().isMerging()) { - // If merges are not possible, we need only run the first unfinished subtrigger - c.trigger().firstUnfinishedSubTrigger().invokeOnElement(c); - } else { - // If merges are possible, we need to run all subtriggers in parallel - for (ExecutableTrigger subTrigger : c.trigger().subTriggers()) { - // Even if the subTrigger is done, it may be revived via merging and must have - // adequate state. - subTrigger.invokeOnElement(c); - } - } - } - - @Override - public void onMerge(OnMergeContext context) throws Exception { - // If merging makes a subtrigger no-longer-finished, it will automatically - // begin participating in shouldFire and onFire appropriately. - - // All the following triggers are retroactively "not started" but that is - // also automatic because they are cleared whenever this trigger - // fires. - boolean priorTriggersAllFinished = true; - for (ExecutableTrigger subTrigger : context.trigger().subTriggers()) { - if (priorTriggersAllFinished) { - subTrigger.invokeOnMerge(context); - priorTriggersAllFinished &= context.forTrigger(subTrigger).trigger().isFinished(); - } else { - subTrigger.invokeClear(context); - } - } - updateFinishedState(context); - } - @Override public Instant getWatermarkThatGuaranteesFiring(BoundedWindow window) { // This trigger will fire at least once when the first trigger in the sequence @@ -105,27 +69,6 @@ public Trigger getContinuationTrigger(List continuationTriggers) { return Repeatedly.forever(new AfterFirst(continuationTriggers)); } - @Override - public boolean shouldFire(Trigger.TriggerContext context) throws Exception { - ExecutableTrigger firstUnfinished = context.trigger().firstUnfinishedSubTrigger(); - return firstUnfinished.invokeShouldFire(context); - } - - @Override - public void onFire(Trigger.TriggerContext context) throws Exception { - context.trigger().firstUnfinishedSubTrigger().invokeOnFire(context); - - // Reset all subtriggers if in a merging context; any may be revived by merging so they are - // all run in parallel for each pending pane. - if (context.trigger().isMerging()) { - for (ExecutableTrigger subTrigger : context.trigger().subTriggers()) { - subTrigger.invokeClear(context); - } - } - - updateFinishedState(context); - } - @Override public String toString() { StringBuilder builder = new StringBuilder("AfterEach.inOrder("); @@ -134,8 +77,4 @@ public String toString() { return builder.toString(); } - - private void updateFinishedState(TriggerContext context) { - context.trigger().setFinished(context.trigger().firstUnfinishedSubTrigger() == null); - } } diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/AfterFirst.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/AfterFirst.java index 7840fc408970..a742b43d7253 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/AfterFirst.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/AfterFirst.java @@ -24,7 +24,6 @@ import java.util.List; import org.apache.beam.sdk.annotations.Experimental; import org.apache.beam.sdk.transforms.windowing.Trigger.OnceTrigger; -import org.apache.beam.sdk.util.ExecutableTrigger; import org.joda.time.Instant; /** @@ -46,21 +45,6 @@ public static AfterFirst of(OnceTrigger... triggers) { return new AfterFirst(Arrays.asList(triggers)); } - @Override - public void onElement(OnElementContext c) throws Exception { - for (ExecutableTrigger subTrigger : c.trigger().subTriggers()) { - subTrigger.invokeOnElement(c); - } - } - - @Override - public void onMerge(OnMergeContext c) throws Exception { - for (ExecutableTrigger subTrigger : c.trigger().subTriggers()) { - subTrigger.invokeOnMerge(c); - } - updateFinishedStatus(c); - } - @Override public Instant getWatermarkThatGuaranteesFiring(BoundedWindow window) { // This trigger will fire after the earliest of its sub-triggers. @@ -79,32 +63,6 @@ public OnceTrigger getContinuationTrigger(List continuationTriggers) { return new AfterFirst(continuationTriggers); } - @Override - public boolean shouldFire(Trigger.TriggerContext context) throws Exception { - for (ExecutableTrigger subtrigger : context.trigger().subTriggers()) { - if (context.forTrigger(subtrigger).trigger().isFinished() - || subtrigger.invokeShouldFire(context)) { - return true; - } - } - return false; - } - - @Override - protected void onOnlyFiring(TriggerContext context) throws Exception { - for (ExecutableTrigger subtrigger : context.trigger().subTriggers()) { - TriggerContext subContext = context.forTrigger(subtrigger); - if (subtrigger.invokeShouldFire(subContext)) { - // If the trigger is ready to fire, then do whatever it needs to do. - subtrigger.invokeOnFire(subContext); - } else { - // If the trigger is not ready to fire, it is nonetheless true that whatever - // pending pane it was tracking is now gone. - subtrigger.invokeClear(subContext); - } - } - } - @Override public String toString() { StringBuilder builder = new StringBuilder("AfterFirst.of("); @@ -113,12 +71,4 @@ public String toString() { return builder.toString(); } - - private void updateFinishedStatus(TriggerContext c) { - boolean anyFinished = false; - for (ExecutableTrigger subTrigger : c.trigger().subTriggers()) { - anyFinished |= c.forTrigger(subTrigger).trigger().isFinished(); - } - c.trigger().setFinished(anyFinished); - } } diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/AfterPane.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/AfterPane.java index 4d59d58539d4..4a706e6648c9 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/AfterPane.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/AfterPane.java @@ -17,7 +17,6 @@ */ package org.apache.beam.sdk.transforms.windowing; -import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import java.util.List; import java.util.Objects; import org.apache.beam.sdk.annotations.Experimental; @@ -25,9 +24,6 @@ import org.apache.beam.sdk.transforms.Sum; import org.apache.beam.sdk.transforms.windowing.Trigger.OnceTrigger; import org.apache.beam.sdk.util.state.AccumulatorCombiningState; -import org.apache.beam.sdk.util.state.MergingStateAccessor; -import org.apache.beam.sdk.util.state.StateAccessor; -import org.apache.beam.sdk.util.state.StateMerging; import org.apache.beam.sdk.util.state.StateTag; import org.apache.beam.sdk.util.state.StateTags; import org.joda.time.Instant; @@ -64,49 +60,6 @@ public static AfterPane elementCountAtLeast(int countElems) { return new AfterPane(countElems); } - @Override - public void onElement(OnElementContext c) throws Exception { - c.state().access(ELEMENTS_IN_PANE_TAG).add(1L); - } - - @Override - public void prefetchOnMerge(MergingStateAccessor state) { - super.prefetchOnMerge(state); - StateMerging.prefetchCombiningValues(state, ELEMENTS_IN_PANE_TAG); - } - - @Override - public void onMerge(OnMergeContext context) throws Exception { - // If we've already received enough elements and finished in some window, - // then this trigger is just finished. - if (context.trigger().finishedInAnyMergingWindow()) { - context.trigger().setFinished(true); - StateMerging.clear(context.state(), ELEMENTS_IN_PANE_TAG); - return; - } - - // Otherwise, compute the sum of elements in all the active panes. - StateMerging.mergeCombiningValues(context.state(), ELEMENTS_IN_PANE_TAG); - } - - @Override - @SuppressFBWarnings(value = "RV_RETURN_VALUE_IGNORED_NO_SIDE_EFFECT", justification = - "prefetch side effect") - public void prefetchShouldFire(StateAccessor state) { - state.access(ELEMENTS_IN_PANE_TAG).readLater(); - } - - @Override - public boolean shouldFire(Trigger.TriggerContext context) throws Exception { - long count = context.state().access(ELEMENTS_IN_PANE_TAG).read(); - return count >= countElems; - } - - @Override - public void clear(TriggerContext c) throws Exception { - c.state().access(ELEMENTS_IN_PANE_TAG).clear(); - } - @Override public boolean isCompatible(Trigger other) { return this.equals(other); @@ -143,9 +96,4 @@ public boolean equals(Object obj) { public int hashCode() { return Objects.hash(countElems); } - - @Override - protected void onOnlyFiring(Trigger.TriggerContext context) throws Exception { - clear(context); - } } diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/AfterProcessingTime.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/AfterProcessingTime.java index f551118b22c0..09f288e63949 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/AfterProcessingTime.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/AfterProcessingTime.java @@ -19,7 +19,6 @@ import java.util.List; import java.util.Objects; -import javax.annotation.Nullable; import org.apache.beam.sdk.annotations.Experimental; import org.apache.beam.sdk.transforms.SerializableFunction; import org.apache.beam.sdk.util.TimeDomain; @@ -36,12 +35,6 @@ @Experimental(Experimental.Kind.TRIGGER) public class AfterProcessingTime extends AfterDelayFromFirstElement { - @Override - @Nullable - public Instant getCurrentTime(Trigger.TriggerContext context) { - return context.currentProcessingTime(); - } - private AfterProcessingTime(List> transforms) { super(TimeDomain.PROCESSING_TIME, transforms); } diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/AfterSynchronizedProcessingTime.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/AfterSynchronizedProcessingTime.java index b96b29344164..b6258f892936 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/AfterSynchronizedProcessingTime.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/AfterSynchronizedProcessingTime.java @@ -20,7 +20,6 @@ import com.google.common.base.Objects; import java.util.Collections; import java.util.List; -import javax.annotation.Nullable; import org.apache.beam.sdk.transforms.SerializableFunction; import org.apache.beam.sdk.util.TimeDomain; import org.joda.time.Instant; @@ -31,12 +30,6 @@ */ public class AfterSynchronizedProcessingTime extends AfterDelayFromFirstElement { - @Override - @Nullable - public Instant getCurrentTime(Trigger.TriggerContext context) { - return context.currentSynchronizedProcessingTime(); - } - public AfterSynchronizedProcessingTime() { super(TimeDomain.SYNCHRONIZED_PROCESSING_TIME, Collections.>emptyList()); diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/AfterWatermark.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/AfterWatermark.java index 89c1ba937576..37b73a6047d4 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/AfterWatermark.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/AfterWatermark.java @@ -25,7 +25,6 @@ import javax.annotation.Nullable; import org.apache.beam.sdk.annotations.Experimental; import org.apache.beam.sdk.transforms.windowing.Trigger.OnceTrigger; -import org.apache.beam.sdk.util.ExecutableTrigger; import org.apache.beam.sdk.util.TimeDomain; import org.joda.time.Instant; @@ -110,50 +109,6 @@ public AfterWatermarkEarlyAndLate withLateFirings(OnceTrigger lateTrigger) { return new AfterWatermarkEarlyAndLate(earlyTrigger, lateTrigger); } - @Override - public void onElement(OnElementContext c) throws Exception { - if (!c.trigger().isMerging()) { - // If merges can never happen, we just run the unfinished subtrigger - c.trigger().firstUnfinishedSubTrigger().invokeOnElement(c); - } else { - // If merges can happen, we run for all subtriggers because they might be - // de-activated or re-activated - for (ExecutableTrigger subTrigger : c.trigger().subTriggers()) { - subTrigger.invokeOnElement(c); - } - } - } - - @Override - public void onMerge(OnMergeContext c) throws Exception { - // NOTE that the ReduceFnRunner will delete all end-of-window timers for the - // merged-away windows. - - ExecutableTrigger earlySubtrigger = c.trigger().subTrigger(EARLY_INDEX); - // We check the early trigger to determine if we are still processing it or - // if the end of window has transitioned us to the late trigger - OnMergeContext earlyContext = c.forTrigger(earlySubtrigger); - - // If the early trigger is still active in any merging window then it is still active in - // the new merged window, because even if the merged window is "done" some pending elements - // haven't had a chance to fire. - if (!earlyContext.trigger().finishedInAllMergingWindows() || !endOfWindowReached(c)) { - earlyContext.trigger().setFinished(false); - if (lateTrigger != null) { - ExecutableTrigger lateSubtrigger = c.trigger().subTrigger(LATE_INDEX); - OnMergeContext lateContext = c.forTrigger(lateSubtrigger); - lateContext.trigger().setFinished(false); - lateSubtrigger.invokeClear(lateContext); - } - } else { - // Otherwise the early trigger and end-of-window bit is done for good. - earlyContext.trigger().setFinished(true); - if (lateTrigger != null) { - c.trigger().subTrigger(LATE_INDEX).invokeOnMerge(c); - } - } - } - @Override public Trigger getContinuationTrigger() { return new AfterWatermarkEarlyAndLate( @@ -173,38 +128,6 @@ public Instant getWatermarkThatGuaranteesFiring(BoundedWindow window) { return window.maxTimestamp(); } - private boolean endOfWindowReached(Trigger.TriggerContext context) { - return context.currentEventTime() != null - && context.currentEventTime().isAfter(context.window().maxTimestamp()); - } - - @Override - public boolean shouldFire(Trigger.TriggerContext context) throws Exception { - if (!context.trigger().isFinished(EARLY_INDEX)) { - // We have not yet transitioned to late firings. - // We should fire if either the trigger is ready or we reach the end of the window. - return context.trigger().subTrigger(EARLY_INDEX).invokeShouldFire(context) - || endOfWindowReached(context); - } else if (lateTrigger == null) { - return false; - } else { - // We are running the late trigger - return context.trigger().subTrigger(LATE_INDEX).invokeShouldFire(context); - } - } - - @Override - public void onFire(Trigger.TriggerContext context) throws Exception { - if (!context.forTrigger(context.trigger().subTrigger(EARLY_INDEX)).trigger().isFinished()) { - onNonLateFiring(context); - } else if (lateTrigger != null) { - onLateFiring(context); - } else { - // all done - context.trigger().setFinished(true); - } - } - @Override public String toString() { StringBuilder builder = new StringBuilder(TO_STRING); @@ -225,47 +148,6 @@ public String toString() { return builder.toString(); } - - private void onNonLateFiring(Trigger.TriggerContext context) throws Exception { - // We have not yet transitioned to late firings. - ExecutableTrigger earlySubtrigger = context.trigger().subTrigger(EARLY_INDEX); - Trigger.TriggerContext earlyContext = context.forTrigger(earlySubtrigger); - - if (!endOfWindowReached(context)) { - // This is an early firing, since we have not arrived at the end of the window - // Implicitly repeats - earlySubtrigger.invokeOnFire(context); - earlySubtrigger.invokeClear(context); - earlyContext.trigger().setFinished(false); - } else { - // We have arrived at the end of the window; terminate the early trigger - // and clear out the late trigger's state - if (earlySubtrigger.invokeShouldFire(context)) { - earlySubtrigger.invokeOnFire(context); - } - earlyContext.trigger().setFinished(true); - earlySubtrigger.invokeClear(context); - - if (lateTrigger == null) { - // Done if there is no late trigger. - context.trigger().setFinished(true); - } else { - // If there is a late trigger, we transition to it, and need to clear its state - // because it was run in parallel. - context.trigger().subTrigger(LATE_INDEX).invokeClear(context); - } - } - - } - - private void onLateFiring(Trigger.TriggerContext context) throws Exception { - // We are firing the late trigger, with implicit repeat - ExecutableTrigger lateSubtrigger = context.trigger().subTrigger(LATE_INDEX); - lateSubtrigger.invokeOnFire(context); - // It is a OnceTrigger, so it must have finished; unfinished it and clear it - lateSubtrigger.invokeClear(context); - context.forTrigger(lateSubtrigger).trigger().setFinished(false); - } } /** @@ -295,33 +177,6 @@ public AfterWatermarkEarlyAndLate withLateFirings(OnceTrigger lateFirings) { return new AfterWatermarkEarlyAndLate(Never.ever(), lateFirings); } - @Override - public void onElement(OnElementContext c) throws Exception { - // We're interested in knowing when the input watermark passes the end of the window. - // (It is possible this has already happened, in which case the timer will be fired - // almost immediately). - c.setTimer(c.window().maxTimestamp(), TimeDomain.EVENT_TIME); - } - - @Override - public void onMerge(OnMergeContext c) throws Exception { - // NOTE that the ReduceFnRunner will delete all end-of-window timers for the - // merged-away windows. - - if (!c.trigger().finishedInAllMergingWindows()) { - // If the trigger is still active in any merging window then it is still active in the new - // merged window, because even if the merged window is "done" some pending elements haven't - // had a chance to fire - c.trigger().setFinished(false); - } else if (!endOfWindowReached(c)) { - // If the end of the new window has not been reached, then the trigger is active again. - c.trigger().setFinished(false); - } else { - // Otherwise it is done for good - c.trigger().setFinished(true); - } - } - @Override public Instant getWatermarkThatGuaranteesFiring(BoundedWindow window) { return window.maxTimestamp(); @@ -346,18 +201,5 @@ public boolean equals(Object obj) { public int hashCode() { return Objects.hash(getClass()); } - - @Override - public boolean shouldFire(Trigger.TriggerContext context) throws Exception { - return endOfWindowReached(context); - } - - private boolean endOfWindowReached(Trigger.TriggerContext context) { - return context.currentEventTime() != null - && context.currentEventTime().isAfter(context.window().maxTimestamp()); - } - - @Override - protected void onOnlyFiring(Trigger.TriggerContext context) throws Exception { } } } diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/DefaultTrigger.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/DefaultTrigger.java index fee7cdfb76df..a649b4ff1f51 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/DefaultTrigger.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/DefaultTrigger.java @@ -19,7 +19,6 @@ import java.util.List; import org.apache.beam.sdk.annotations.Experimental; -import org.apache.beam.sdk.util.TimeDomain; import org.joda.time.Instant; /** @@ -40,27 +39,6 @@ public static DefaultTrigger of() { return new DefaultTrigger(); } - @Override - public void onElement(OnElementContext c) throws Exception { - // If the end of the window has already been reached, then we are already ready to fire - // and do not need to set a wake-up timer. - if (!endOfWindowReached(c)) { - c.setTimer(c.window().maxTimestamp(), TimeDomain.EVENT_TIME); - } - } - - @Override - public void onMerge(OnMergeContext c) throws Exception { - // If the end of the window has already been reached, then we are already ready to fire - // and do not need to set a wake-up timer. - if (!endOfWindowReached(c)) { - c.setTimer(c.window().maxTimestamp(), TimeDomain.EVENT_TIME); - } - } - - @Override - public void clear(TriggerContext c) throws Exception { } - @Override public Instant getWatermarkThatGuaranteesFiring(BoundedWindow window) { return window.maxTimestamp(); @@ -76,17 +54,4 @@ public boolean isCompatible(Trigger other) { public Trigger getContinuationTrigger(List continuationTriggers) { return this; } - - @Override - public boolean shouldFire(Trigger.TriggerContext context) throws Exception { - return endOfWindowReached(context); - } - - private boolean endOfWindowReached(Trigger.TriggerContext context) { - return context.currentEventTime() != null - && context.currentEventTime().isAfter(context.window().maxTimestamp()); - } - - @Override - public void onFire(Trigger.TriggerContext context) throws Exception { } } diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/Never.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/Never.java index 07b70f4ee389..664ae83460e8 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/Never.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/Never.java @@ -47,12 +47,6 @@ protected NeverTrigger() { super(null); } - @Override - public void onElement(OnElementContext c) {} - - @Override - public void onMerge(OnMergeContext c) {} - @Override protected Trigger getContinuationTrigger(List continuationTriggers) { return this; @@ -62,16 +56,5 @@ protected Trigger getContinuationTrigger(List continuationTriggers) { public Instant getWatermarkThatGuaranteesFiring(BoundedWindow window) { return BoundedWindow.TIMESTAMP_MAX_VALUE; } - - @Override - public boolean shouldFire(Trigger.TriggerContext context) { - return false; - } - - @Override - protected void onOnlyFiring(Trigger.TriggerContext context) { - throw new UnsupportedOperationException( - String.format("%s should never fire", getClass().getSimpleName())); - } } } diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/OrFinallyTrigger.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/OrFinallyTrigger.java index 9bef45ad3a11..1ed9b550ca99 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/OrFinallyTrigger.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/OrFinallyTrigger.java @@ -20,7 +20,6 @@ import com.google.common.annotations.VisibleForTesting; import java.util.Arrays; import java.util.List; -import org.apache.beam.sdk.util.ExecutableTrigger; import org.joda.time.Instant; /** @@ -50,20 +49,6 @@ public OnceTrigger getUntilTrigger() { return (OnceTrigger) subTriggers().get(UNTIL); } - @Override - public void onElement(OnElementContext c) throws Exception { - c.trigger().subTrigger(ACTUAL).invokeOnElement(c); - c.trigger().subTrigger(UNTIL).invokeOnElement(c); - } - - @Override - public void onMerge(OnMergeContext c) throws Exception { - for (ExecutableTrigger subTrigger : c.trigger().subTriggers()) { - subTrigger.invokeOnMerge(c); - } - updateFinishedState(c); - } - @Override public Instant getWatermarkThatGuaranteesFiring(BoundedWindow window) { // This trigger fires once either the trigger or the until trigger fires. @@ -82,39 +67,8 @@ public Trigger getContinuationTrigger(List continuationTriggers) { (Trigger.OnceTrigger) continuationTriggers.get(UNTIL))); } - @Override - public boolean shouldFire(Trigger.TriggerContext context) throws Exception { - return context.trigger().subTrigger(ACTUAL).invokeShouldFire(context) - || context.trigger().subTrigger(UNTIL).invokeShouldFire(context); - } - - @Override - public void onFire(Trigger.TriggerContext context) throws Exception { - ExecutableTrigger actualSubtrigger = context.trigger().subTrigger(ACTUAL); - ExecutableTrigger untilSubtrigger = context.trigger().subTrigger(UNTIL); - - if (untilSubtrigger.invokeShouldFire(context)) { - untilSubtrigger.invokeOnFire(context); - actualSubtrigger.invokeClear(context); - } else { - // If until didn't fire, then the actual must have (or it is forbidden to call - // onFire) so we are done only if actual is done. - actualSubtrigger.invokeOnFire(context); - // Do not clear the until trigger, because it tracks data cross firings. - } - updateFinishedState(context); - } - @Override public String toString() { return String.format("%s.orFinally(%s)", subTriggers.get(ACTUAL), subTriggers.get(UNTIL)); } - - private void updateFinishedState(TriggerContext c) throws Exception { - boolean anyStillFinished = false; - for (ExecutableTrigger subTrigger : c.trigger().subTriggers()) { - anyStillFinished |= c.forTrigger(subTrigger).trigger().isFinished(); - } - c.trigger().setFinished(anyStillFinished); - } } diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/Repeatedly.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/Repeatedly.java index 45bc6c128ef6..4d79a2c22910 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/Repeatedly.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/Repeatedly.java @@ -19,7 +19,6 @@ import java.util.Arrays; import java.util.List; -import org.apache.beam.sdk.util.ExecutableTrigger; import org.joda.time.Instant; /** @@ -60,16 +59,6 @@ public Trigger getRepeatedTrigger() { return repeatedTrigger; } - @Override - public void onElement(OnElementContext c) throws Exception { - getRepeated(c).invokeOnElement(c); - } - - @Override - public void onMerge(OnMergeContext c) throws Exception { - getRepeated(c).invokeOnMerge(c); - } - @Override public Instant getWatermarkThatGuaranteesFiring(BoundedWindow window) { // This trigger fires once the repeated trigger fires. @@ -81,27 +70,8 @@ public Trigger getContinuationTrigger(List continuationTriggers) { return new Repeatedly(continuationTriggers.get(REPEATED)); } - @Override - public boolean shouldFire(Trigger.TriggerContext context) throws Exception { - return getRepeated(context).invokeShouldFire(context); - } - - @Override - public void onFire(TriggerContext context) throws Exception { - getRepeated(context).invokeOnFire(context); - - if (context.trigger().isFinished(REPEATED)) { - // Reset tree will recursively clear the finished bits, and invoke clear. - context.forTrigger(getRepeated(context)).trigger().resetTree(); - } - } - @Override public String toString() { return String.format("Repeatedly.forever(%s)", subTriggers.get(REPEATED)); } - - private ExecutableTrigger getRepeated(TriggerContext context) { - return context.trigger().subTrigger(REPEATED); - } } diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/Trigger.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/Trigger.java index 18b7a6291053..1cc807e85adb 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/Trigger.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/Trigger.java @@ -23,22 +23,18 @@ import java.util.Collections; import java.util.List; import java.util.Objects; -import javax.annotation.Nullable; import org.apache.beam.sdk.annotations.Experimental; -import org.apache.beam.sdk.util.ExecutableTrigger; -import org.apache.beam.sdk.util.TimeDomain; -import org.apache.beam.sdk.util.state.MergingStateAccessor; -import org.apache.beam.sdk.util.state.StateAccessor; +import org.apache.beam.sdk.transforms.GroupByKey; import org.joda.time.Instant; /** - * {@code Trigger}s control when the elements for a specific key and window are output. As elements - * arrive, they are put into one or more windows by a {@link Window} transform and its associated - * {@link WindowFn}, and then passed to the associated {@code Trigger} to determine if the - * {@code Window}s contents should be output. + * {@link Trigger Triggers} control when the elements for a specific key and window are output. As + * elements arrive, they are put into one or more windows by a {@link Window} transform and its + * associated {@link WindowFn}, and then passed to the associated {@link Trigger} to determine if + * the {@link BoundedWindow Window's} contents should be output. * - *

See {@link org.apache.beam.sdk.transforms.GroupByKey} and {@link Window} - * for more information about how grouping with windows works. + *

See {@link GroupByKey} and {@link Window} for more information about how grouping with windows + * works. * *

The elements that are assigned to a window since the last time it was fired (or since the * window was created) are placed into the current window pane. Triggers are evaluated against the @@ -46,224 +42,34 @@ * output. When the root trigger finishes (indicating it will never fire again), the window is * closed and any new elements assigned to that window are discarded. * - *

Several predefined {@code Trigger}s are provided: + *

Several predefined {@link Trigger Triggers} are provided: + * *

    - *
  • {@link AfterWatermark} for firing when the watermark passes a timestamp determined from - * either the end of the window or the arrival of the first element in a pane. - *
  • {@link AfterProcessingTime} for firing after some amount of processing time has elapsed - * (typically since the first element in a pane). - *
  • {@link AfterPane} for firing off a property of the elements in the current pane, such as - * the number of elements that have been assigned to the current pane. + *
  • {@link AfterWatermark} for firing when the watermark passes a timestamp determined from + * either the end of the window or the arrival of the first element in a pane. + *
  • {@link AfterProcessingTime} for firing after some amount of processing time has elapsed + * (typically since the first element in a pane). + *
  • {@link AfterPane} for firing off a property of the elements in the current pane, such as the + * number of elements that have been assigned to the current pane. *
* *

In addition, {@code Trigger}s can be combined in a variety of ways: - *

    - *
  • {@link Repeatedly#forever} to create a trigger that executes forever. Any time its - * argument finishes it gets reset and starts over. Can be combined with - * {@link Trigger#orFinally} to specify a condition that causes the repetition to stop. - *
  • {@link AfterEach#inOrder} to execute each trigger in sequence, firing each (and every) - * time that a trigger fires, and advancing to the next trigger in the sequence when it finishes. - *
  • {@link AfterFirst#of} to create a trigger that fires after at least one of its arguments - * fires. An {@link AfterFirst} trigger finishes after it fires once. - *
  • {@link AfterAll#of} to create a trigger that fires after all least one of its arguments - * have fired at least once. An {@link AfterAll} trigger finishes after it fires once. - *
* - *

Each trigger tree is instantiated per-key and per-window. Every trigger in the tree is in one - * of the following states: *

    - *
  • Never Existed - before the trigger has started executing, there is no state associated - * with it anywhere in the system. A trigger moves to the executing state as soon as it - * processes in the current pane. - *
  • Executing - while the trigger is receiving items and may fire. While it is in this state, - * it may persist book-keeping information to persisted state, set timers, etc. - *
  • Finished - after a trigger finishes, all of its book-keeping data is cleaned up, and the - * system remembers only that it is finished. Entering this state causes us to discard any - * elements in the buffer for that window, as well. + *
  • {@link Repeatedly#forever} to create a trigger that executes forever. Any time its argument + * finishes it gets reset and starts over. Can be combined with {@link Trigger#orFinally} to + * specify a condition that causes the repetition to stop. + *
  • {@link AfterEach#inOrder} to execute each trigger in sequence, firing each (and every) time + * that a trigger fires, and advancing to the next trigger in the sequence when it finishes. + *
  • {@link AfterFirst#of} to create a trigger that fires after at least one of its arguments + * fires. An {@link AfterFirst} trigger finishes after it fires once. + *
  • {@link AfterAll#of} to create a trigger that fires after all least one of its arguments have + * fired at least once. An {@link AfterAll} trigger finishes after it fires once. *
- * - *

Once finished, a trigger cannot return itself back to an earlier state, however a composite - * trigger could reset its sub-triggers. - * - *

Triggers should not build up any state internally since they may be recreated - * between invocations of the callbacks. All important values should be persisted using - * state before the callback returns. */ @Experimental(Experimental.Kind.TRIGGER) public abstract class Trigger implements Serializable { - /** - * Interface for accessing information about the trigger being executed and other triggers in the - * same tree. - */ - public interface TriggerInfo { - - /** - * Returns true if the windowing strategy of the current {@code PCollection} is a merging - * WindowFn. If true, the trigger execution needs to keep enough information to support the - * possibility of {@link Trigger#onMerge} being called. If false, {@link Trigger#onMerge} will - * never be called. - */ - boolean isMerging(); - - /** - * Access the executable versions of the sub-triggers of the current trigger. - */ - Iterable subTriggers(); - - /** - * Access the executable version of the specified sub-trigger. - */ - ExecutableTrigger subTrigger(int subtriggerIndex); - - /** - * Returns true if the current trigger is marked finished. - */ - boolean isFinished(); - - /** - * Return true if the given subtrigger is marked finished. - */ - boolean isFinished(int subtriggerIndex); - - /** - * Returns true if all the sub-triggers of the current trigger are marked finished. - */ - boolean areAllSubtriggersFinished(); - - /** - * Returns an iterable over the unfinished sub-triggers of the current trigger. - */ - Iterable unfinishedSubTriggers(); - - /** - * Returns the first unfinished sub-trigger. - */ - ExecutableTrigger firstUnfinishedSubTrigger(); - - /** - * Clears all keyed state for triggers in the current sub-tree and unsets all the associated - * finished bits. - */ - void resetTree() throws Exception; - - /** - * Sets the finished bit for the current trigger. - */ - void setFinished(boolean finished); - - /** - * Sets the finished bit for the given sub-trigger. - */ - void setFinished(boolean finished, int subTriggerIndex); - } - - /** - * Interact with properties of the trigger being executed, with extensions to deal with the - * merging windows. - */ - public interface MergingTriggerInfo extends TriggerInfo { - - /** Return true if the trigger is finished in any window being merged. */ - boolean finishedInAnyMergingWindow(); - - /** Return true if the trigger is finished in all windows being merged. */ - boolean finishedInAllMergingWindows(); - } - - /** - * Information accessible to all operational hooks in this {@code Trigger}. - * - *

Used directly in {@link Trigger#shouldFire} and {@link Trigger#clear}, and - * extended with additional information in other methods. - */ - public abstract class TriggerContext { - - /** Returns the interface for accessing trigger info. */ - public abstract TriggerInfo trigger(); - - /** Returns the interface for accessing persistent state. */ - public abstract StateAccessor state(); - - /** The window that the current context is executing in. */ - public abstract BoundedWindow window(); - - /** Create a sub-context for the given sub-trigger. */ - public abstract TriggerContext forTrigger(ExecutableTrigger trigger); - - /** - * Removes the timer set in this trigger context for the given {@link Instant} - * and {@link TimeDomain}. - */ - public abstract void deleteTimer(Instant timestamp, TimeDomain domain); - - /** The current processing time. */ - public abstract Instant currentProcessingTime(); - - /** The current synchronized upstream processing time or {@code null} if unknown. */ - @Nullable - public abstract Instant currentSynchronizedProcessingTime(); - - /** The current event time for the input or {@code null} if unknown. */ - @Nullable - public abstract Instant currentEventTime(); - } - - /** - * Extended {@link TriggerContext} containing information accessible to the {@link #onElement} - * operational hook. - */ - public abstract class OnElementContext extends TriggerContext { - /** The event timestamp of the element currently being processed. */ - public abstract Instant eventTimestamp(); - - /** - * Sets a timer to fire when the watermark or processing time is beyond the given timestamp. - * Timers are not guaranteed to fire immediately, but will be delivered at some time afterwards. - * - *

As with {@link #state}, timers are implicitly scoped to the current window. All - * timer firings for a window will be received, but the implementation should choose to ignore - * those that are not applicable. - * - * @param timestamp the time at which the trigger should be re-evaluated - * @param domain the domain that the {@code timestamp} applies to - */ - public abstract void setTimer(Instant timestamp, TimeDomain domain); - - /** Create an {@code OnElementContext} for executing the given trigger. */ - @Override - public abstract OnElementContext forTrigger(ExecutableTrigger trigger); - } - - /** - * Extended {@link TriggerContext} containing information accessible to the {@link #onMerge} - * operational hook. - */ - public abstract class OnMergeContext extends TriggerContext { - /** - * Sets a timer to fire when the watermark or processing time is beyond the given timestamp. - * Timers are not guaranteed to fire immediately, but will be delivered at some time afterwards. - * - *

As with {@link #state}, timers are implicitly scoped to the current window. All - * timer firings for a window will be received, but the implementation should choose to ignore - * those that are not applicable. - * - * @param timestamp the time at which the trigger should be re-evaluated - * @param domain the domain that the {@code timestamp} applies to - */ - public abstract void setTimer(Instant timestamp, TimeDomain domain); - - /** Create an {@code OnMergeContext} for executing the given trigger. */ - @Override - public abstract OnMergeContext forTrigger(ExecutableTrigger trigger); - - @Override - public abstract MergingStateAccessor state(); - - @Override - public abstract MergingTriggerInfo trigger(); - } - protected final List subTriggers; protected Trigger(List subTriggers) { @@ -274,114 +80,14 @@ protected Trigger() { this(Collections.EMPTY_LIST); } - /** - * Called every time an element is incorporated into a window. - */ - public abstract void onElement(OnElementContext c) throws Exception; - - /** - * Called immediately after windows have been merged. - * - *

Leaf triggers should update their state by inspecting their status and any state - * in the merging windows. Composite triggers should update their state by calling - * {@link ExecutableTrigger#invokeOnMerge} on their sub-triggers, and applying appropriate logic. - * - *

A trigger such as {@link AfterWatermark#pastEndOfWindow} may no longer be finished; - * it is the responsibility of the trigger itself to record this fact. It is forbidden for - * a trigger to become finished due to {@link #onMerge}, as it has not yet fired the pending - * elements that led to it being ready to fire. - * - *

The implementation does not need to clear out any state associated with the old windows. - */ - public abstract void onMerge(OnMergeContext c) throws Exception; - - /** - * Returns {@code true} if the current state of the trigger indicates that its condition - * is satisfied and it is ready to fire. - */ - public abstract boolean shouldFire(TriggerContext context) throws Exception; - - /** - * Adjusts the state of the trigger to be ready for the next pane. For example, a - * {@link Repeatedly} trigger will reset its inner trigger, since it has fired. - * - *

If the trigger is finished, it is the responsibility of the trigger itself to - * record that fact via the {@code context}. - */ - public abstract void onFire(TriggerContext context) throws Exception; - - /** - * Called to allow the trigger to prefetch any state it will likely need to read from during - * an {@link #onElement} call. - */ - public void prefetchOnElement(StateAccessor state) { - if (subTriggers != null) { - for (Trigger trigger : subTriggers) { - trigger.prefetchOnElement(state); - } - } - } - - /** - * Called to allow the trigger to prefetch any state it will likely need to read from during - * an {@link #onMerge} call. - */ - public void prefetchOnMerge(MergingStateAccessor state) { - if (subTriggers != null) { - for (Trigger trigger : subTriggers) { - trigger.prefetchOnMerge(state); - } - } - } - - /** - * Called to allow the trigger to prefetch any state it will likely need to read from during - * an {@link #shouldFire} call. - */ - public void prefetchShouldFire(StateAccessor state) { - if (subTriggers != null) { - for (Trigger trigger : subTriggers) { - trigger.prefetchShouldFire(state); - } - } - } - - /** - * Called to allow the trigger to prefetch any state it will likely need to read from during - * an {@link #onFire} call. - */ - public void prefetchOnFire(StateAccessor state) { - if (subTriggers != null) { - for (Trigger trigger : subTriggers) { - trigger.prefetchOnFire(state); - } - } - } - - /** - * Clear any state associated with this trigger in the given window. - * - *

This is called after a trigger has indicated it will never fire again. The trigger system - * keeps enough information to know that the trigger is finished, so this trigger should clear all - * of its state. - */ - public void clear(TriggerContext c) throws Exception { - if (subTriggers != null) { - for (ExecutableTrigger trigger : c.trigger().subTriggers()) { - trigger.invokeClear(c); - } - } - } - public List subTriggers() { return subTriggers; } /** - * Return a trigger to use after a {@code GroupByKey} to preserve the - * intention of this trigger. Specifically, triggers that are time based - * and intended to provide speculative results should continue providing - * speculative results. Triggers that fire once (or multiple times) should + * Return a trigger to use after a {@link GroupByKey} to preserve the intention of this trigger. + * Specifically, triggers that are time based and intended to provide speculative results should + * continue providing speculative results. Triggers that fire once (or multiple times) should * continue firing once (or multiple times). */ public Trigger getContinuationTrigger() { @@ -397,27 +103,24 @@ public Trigger getContinuationTrigger() { } /** - * Return the {@link #getContinuationTrigger} of this {@code Trigger}. For convenience, this - * is provided the continuation trigger of each of the sub-triggers. + * Return the {@link #getContinuationTrigger} of this {@link Trigger}. For convenience, this is + * provided the continuation trigger of each of the sub-triggers. */ protected abstract Trigger getContinuationTrigger(List continuationTriggers); /** - * Returns a bound in watermark time by which this trigger would have fired at least once - * for a given window had there been input data. This is a static property of a trigger - * that does not depend on its state. + * Returns a bound in event time by which this trigger would have fired at least once for a given + * window had there been input data. * - *

For triggers that do not fire based on the watermark advancing, returns - * {@link BoundedWindow#TIMESTAMP_MAX_VALUE}. + *

For triggers that do not fire based on the watermark advancing, returns {@link + * BoundedWindow#TIMESTAMP_MAX_VALUE}. * - *

This estimate is used to determine that there are no elements in a side-input window, which - * causes the default value to be used instead. + *

This estimate may be used, for example, to determine that there are no elements in a + * side-input window, which causes the default value to be used instead. */ public abstract Instant getWatermarkThatGuaranteesFiring(BoundedWindow window); - /** - * Returns whether this performs the same triggering as the given {@code Trigger}. - */ + /** Returns whether this performs the same triggering as the given {@link Trigger}. */ public boolean isCompatible(Trigger other) { if (!getClass().equals(other.getClass())) { return false; @@ -472,31 +175,33 @@ public int hashCode() { } /** - * Specify an ending condition for this trigger. If the {@code until} fires then the combination - * fires. + * Specify an ending condition for this trigger. If the {@code until} {@link Trigger} fires then + * the combination fires. * - *

The expression {@code t1.orFinally(t2)} fires every time {@code t1} fires, and finishes - * as soon as either {@code t1} finishes or {@code t2} fires, in which case it fires one last time - * for {@code t2}. Both {@code t1} and {@code t2} are executed in parallel. This means that - * {@code t1} may have fired since {@code t2} started, so not all of the elements that {@code t2} - * has seen are necessarily in the current pane. + *

The expression {@code t1.orFinally(t2)} fires every time {@code t1} fires, and finishes as + * soon as either {@code t1} finishes or {@code t2} fires, in which case it fires one last time + * for {@code t2}. Both {@code t1} and {@code t2} are executed in parallel. This means that {@code + * t1} may have fired since {@code t2} started, so not all of the elements that {@code t2} has + * seen are necessarily in the current pane. * *

For example the final firing of the following trigger may only have 1 element: - * 

 {@code
+   *
+   * 
{@code
    * Repeatedly.forever(AfterPane.elementCountAtLeast(2))
    *     .orFinally(AfterPane.elementCountAtLeast(5))
-   * } 

+   * }
+   *
* - *

Note that if {@code t1} is {@link OnceTrigger}, then {@code t1.orFinally(t2)} is the same - * as {@code AfterFirst.of(t1, t2)}. + *

Note that if {@code t1} is {@link OnceTrigger}, then {@code t1.orFinally(t2)} is the same as + * {@code AfterFirst.of(t1, t2)}. */ public OrFinallyTrigger orFinally(OnceTrigger until) { return new OrFinallyTrigger(this, until); } /** - * {@link Trigger}s that are guaranteed to fire at most once should extend from this, rather - * than the general {@link Trigger} class to indicate that behavior. + * {@link Trigger Triggers} that are guaranteed to fire at most once should extend {@link + * OnceTrigger} rather than the general {@link Trigger} class to indicate that behavior. */ public abstract static class OnceTrigger extends Trigger { protected OnceTrigger(List subTriggers) { @@ -511,20 +216,5 @@ public final OnceTrigger getContinuationTrigger() { } return (OnceTrigger) continuation; } - - /** - * {@inheritDoc} - */ - @Override - public final void onFire(TriggerContext context) throws Exception { - onOnlyFiring(context); - context.trigger().setFinished(true); - } - - /** - * Called exactly once by {@link #onFire} when the trigger is fired. By default, - * invokes {@link #onFire} on all subtriggers for which {@link #shouldFire} is {@code true}. - */ - protected abstract void onOnlyFiring(TriggerContext context) throws Exception; } } diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/ExecutableTrigger.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/ExecutableTrigger.java index 088c499daa0b..48a49aa493a6 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/ExecutableTrigger.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/ExecutableTrigger.java @@ -29,9 +29,13 @@ /** * A wrapper around a trigger used during execution. While an actual trigger may appear multiple - * times (both in the same trigger expression and in other trigger expressions), the - * {@code ExecutableTrigger} wrapped around them forms a tree (only one occurrence). + * times (both in the same trigger expression and in other trigger expressions), the {@code + * ExecutableTrigger} wrapped around them forms a tree (only one occurrence). + * + * @deprecated uses of {@link ExecutableTrigger} should be ported to + * org.apache.beam.runners.core.triggers.ExecutableTriggerStateMachine. */ +@Deprecated public class ExecutableTrigger implements Serializable { /** Store the index assigned to this trigger. */ @@ -114,38 +118,6 @@ public ExecutableTrigger getSubTriggerContaining(int index) { return previous; } - /** - * Invoke the {@link Trigger#onElement} method for this trigger, ensuring that the bits are - * properly updated if the trigger finishes. - */ - public void invokeOnElement(Trigger.OnElementContext c) throws Exception { - trigger.onElement(c.forTrigger(this)); - } - - /** - * Invoke the {@link Trigger#onMerge} method for this trigger, ensuring that the bits are properly - * updated. - */ - public void invokeOnMerge(Trigger.OnMergeContext c) throws Exception { - Trigger.OnMergeContext subContext = c.forTrigger(this); - trigger.onMerge(subContext); - } - - public boolean invokeShouldFire(Trigger.TriggerContext c) throws Exception { - return trigger.shouldFire(c.forTrigger(this)); - } - - public void invokeOnFire(Trigger.TriggerContext c) throws Exception { - trigger.onFire(c.forTrigger(this)); - } - - /** - * Invoke clear for the current this trigger. - */ - public void invokeClear(Trigger.TriggerContext c) throws Exception { - trigger.clear(c.forTrigger(this)); - } - /** * {@link ExecutableTrigger} that enforces the fact that the trigger should always FIRE_AND_FINISH * and never just FIRE. diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/FinishedTriggers.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/FinishedTriggers.java deleted file mode 100644 index ea14c40a51fb..000000000000 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/FinishedTriggers.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.beam.sdk.util; - -/** - * A mutable set which tracks whether any particular {@link ExecutableTrigger} is - * finished. - */ -public interface FinishedTriggers { - /** - * Returns {@code true} if the trigger is finished. - */ - boolean isFinished(ExecutableTrigger trigger); - - /** - * Sets the fact that the trigger is finished. - */ - void setFinished(ExecutableTrigger trigger, boolean value); - - /** - * Sets the trigger and all of its subtriggers to unfinished. - */ - void clearRecursively(ExecutableTrigger trigger); - - /** - * Create an independent copy of this mutable {@link FinishedTriggers}. - */ - FinishedTriggers copy(); -} diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/FinishedTriggersBitSet.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/FinishedTriggersBitSet.java deleted file mode 100644 index 4cd617fca348..000000000000 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/FinishedTriggersBitSet.java +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.beam.sdk.util; - -import java.util.BitSet; - -/** - * A {@link FinishedTriggers} implementation based on an underlying {@link BitSet}. - */ -public class FinishedTriggersBitSet implements FinishedTriggers { - - private final BitSet bitSet; - - private FinishedTriggersBitSet(BitSet bitSet) { - this.bitSet = bitSet; - } - - public static FinishedTriggersBitSet emptyWithCapacity(int capacity) { - return new FinishedTriggersBitSet(new BitSet(capacity)); - } - - public static FinishedTriggersBitSet fromBitSet(BitSet bitSet) { - return new FinishedTriggersBitSet(bitSet); - } - - /** - * Returns the underlying {@link BitSet} for this {@link FinishedTriggersBitSet}. - */ - public BitSet getBitSet() { - return bitSet; - } - - @Override - public boolean isFinished(ExecutableTrigger trigger) { - return bitSet.get(trigger.getTriggerIndex()); - } - - @Override - public void setFinished(ExecutableTrigger trigger, boolean value) { - bitSet.set(trigger.getTriggerIndex(), value); - } - - @Override - public void clearRecursively(ExecutableTrigger trigger) { - bitSet.clear(trigger.getTriggerIndex(), trigger.getFirstIndexAfterSubtree()); - } - - @Override - public FinishedTriggersBitSet copy() { - return new FinishedTriggersBitSet((BitSet) bitSet.clone()); - } -} diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/FinishedTriggersSet.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/FinishedTriggersSet.java deleted file mode 100644 index a9feb7300a17..000000000000 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/FinishedTriggersSet.java +++ /dev/null @@ -1,72 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.beam.sdk.util; - -import com.google.common.collect.Sets; -import java.util.Set; - -/** - * An implementation of {@link FinishedTriggers} atop a user-provided mutable {@link Set}. - */ -public class FinishedTriggersSet implements FinishedTriggers { - - private final Set finishedTriggers; - - private FinishedTriggersSet(Set finishedTriggers) { - this.finishedTriggers = finishedTriggers; - } - - public static FinishedTriggersSet fromSet(Set finishedTriggers) { - return new FinishedTriggersSet(finishedTriggers); - } - - /** - * Returns a mutable {@link Set} of the underlying triggers that are finished. - */ - public Set getFinishedTriggers() { - return finishedTriggers; - } - - @Override - public boolean isFinished(ExecutableTrigger trigger) { - return finishedTriggers.contains(trigger); - } - - @Override - public void setFinished(ExecutableTrigger trigger, boolean value) { - if (value) { - finishedTriggers.add(trigger); - } else { - finishedTriggers.remove(trigger); - } - } - - @Override - public void clearRecursively(ExecutableTrigger trigger) { - finishedTriggers.remove(trigger); - for (ExecutableTrigger subTrigger : trigger.subTriggers()) { - clearRecursively(subTrigger); - } - } - - @Override - public FinishedTriggersSet copy() { - return fromSet(Sets.newHashSet(finishedTriggers)); - } - -} diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/ReshuffleTrigger.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/ReshuffleTrigger.java index 437f14a73ea0..8dd648adcd50 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/ReshuffleTrigger.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/ReshuffleTrigger.java @@ -34,12 +34,6 @@ public ReshuffleTrigger() { super(); } - @Override - public void onElement(Trigger.OnElementContext c) { } - - @Override - public void onMerge(Trigger.OnMergeContext c) { } - @Override protected Trigger getContinuationTrigger(List continuationTriggers) { return this; @@ -51,14 +45,6 @@ public Instant getWatermarkThatGuaranteesFiring(BoundedWindow window) { "ReshuffleTrigger should not be used outside of Reshuffle"); } - @Override - public boolean shouldFire(Trigger.TriggerContext context) throws Exception { - return true; - } - - @Override - public void onFire(Trigger.TriggerContext context) throws Exception { } - @Override public String toString() { return "ReshuffleTrigger()"; diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/TriggerContextFactory.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/TriggerContextFactory.java deleted file mode 100644 index e09aac2eee75..000000000000 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/TriggerContextFactory.java +++ /dev/null @@ -1,507 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.beam.sdk.util; - -import com.google.common.base.Predicate; -import com.google.common.collect.FluentIterable; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.Iterables; -import java.util.Collection; -import java.util.Map; -import javax.annotation.Nullable; -import org.apache.beam.sdk.coders.Coder; -import org.apache.beam.sdk.transforms.windowing.BoundedWindow; -import org.apache.beam.sdk.transforms.windowing.Trigger; -import org.apache.beam.sdk.transforms.windowing.Trigger.MergingTriggerInfo; -import org.apache.beam.sdk.transforms.windowing.Trigger.TriggerInfo; -import org.apache.beam.sdk.transforms.windowing.WindowFn; -import org.apache.beam.sdk.util.state.MergingStateAccessor; -import org.apache.beam.sdk.util.state.State; -import org.apache.beam.sdk.util.state.StateAccessor; -import org.apache.beam.sdk.util.state.StateInternals; -import org.apache.beam.sdk.util.state.StateNamespace; -import org.apache.beam.sdk.util.state.StateNamespaces; -import org.apache.beam.sdk.util.state.StateTag; -import org.joda.time.Instant; - -/** - * Factory for creating instances of the various {@link Trigger} contexts. - * - *

These contexts are highly interdependent and share many fields; it is inadvisable - * to create them via any means other than this factory class. - */ -public class TriggerContextFactory { - - private final WindowFn windowFn; - private StateInternals stateInternals; - private final Coder windowCoder; - - public TriggerContextFactory(WindowFn windowFn, - StateInternals stateInternals, ActiveWindowSet activeWindows) { - // Future triggers may be able to exploit the active window to state address window mapping. - this.windowFn = windowFn; - this.stateInternals = stateInternals; - this.windowCoder = windowFn.windowCoder(); - } - - public Trigger.TriggerContext base(W window, Timers timers, - ExecutableTrigger rootTrigger, FinishedTriggers finishedSet) { - return new TriggerContextImpl(window, timers, rootTrigger, finishedSet); - } - - public Trigger.OnElementContext createOnElementContext( - W window, Timers timers, Instant elementTimestamp, - ExecutableTrigger rootTrigger, FinishedTriggers finishedSet) { - return new OnElementContextImpl(window, timers, rootTrigger, finishedSet, elementTimestamp); - } - - public Trigger.OnMergeContext createOnMergeContext(W window, Timers timers, - ExecutableTrigger rootTrigger, FinishedTriggers finishedSet, - Map finishedSets) { - return new OnMergeContextImpl(window, timers, rootTrigger, finishedSet, finishedSets); - } - - public StateAccessor createStateAccessor(W window, ExecutableTrigger trigger) { - return new StateAccessorImpl(window, trigger); - } - - public MergingStateAccessor createMergingStateAccessor( - W mergeResult, Collection mergingWindows, ExecutableTrigger trigger) { - return new MergingStateAccessorImpl(trigger, mergingWindows, mergeResult); - } - - private class TriggerInfoImpl implements Trigger.TriggerInfo { - - protected final ExecutableTrigger trigger; - protected final FinishedTriggers finishedSet; - private final Trigger.TriggerContext context; - - public TriggerInfoImpl(ExecutableTrigger trigger, FinishedTriggers finishedSet, - Trigger.TriggerContext context) { - this.trigger = trigger; - this.finishedSet = finishedSet; - this.context = context; - } - - @Override - public boolean isMerging() { - return !windowFn.isNonMerging(); - } - - @Override - public Iterable subTriggers() { - return trigger.subTriggers(); - } - - @Override - public ExecutableTrigger subTrigger(int subtriggerIndex) { - return trigger.subTriggers().get(subtriggerIndex); - } - - @Override - public boolean isFinished() { - return finishedSet.isFinished(trigger); - } - - @Override - public boolean isFinished(int subtriggerIndex) { - return finishedSet.isFinished(subTrigger(subtriggerIndex)); - } - - @Override - public boolean areAllSubtriggersFinished() { - return Iterables.isEmpty(unfinishedSubTriggers()); - } - - @Override - public Iterable unfinishedSubTriggers() { - return FluentIterable - .from(trigger.subTriggers()) - .filter(new Predicate() { - @Override - public boolean apply(ExecutableTrigger trigger) { - return !finishedSet.isFinished(trigger); - } - }); - } - - @Override - public ExecutableTrigger firstUnfinishedSubTrigger() { - for (ExecutableTrigger subTrigger : trigger.subTriggers()) { - if (!finishedSet.isFinished(subTrigger)) { - return subTrigger; - } - } - return null; - } - - @Override - public void resetTree() throws Exception { - finishedSet.clearRecursively(trigger); - trigger.invokeClear(context); - } - - @Override - public void setFinished(boolean finished) { - finishedSet.setFinished(trigger, finished); - } - - @Override - public void setFinished(boolean finished, int subTriggerIndex) { - finishedSet.setFinished(subTrigger(subTriggerIndex), finished); - } - } - - private class TriggerTimers implements Timers { - - private final Timers timers; - private final W window; - - public TriggerTimers(W window, Timers timers) { - this.timers = timers; - this.window = window; - } - - @Override - public void setTimer(Instant timestamp, TimeDomain timeDomain) { - timers.setTimer(timestamp, timeDomain); - } - - @Override - public void deleteTimer(Instant timestamp, TimeDomain timeDomain) { - if (timeDomain == TimeDomain.EVENT_TIME - && timestamp.equals(window.maxTimestamp())) { - // Don't allow triggers to unset the at-max-timestamp timer. This is necessary for on-time - // state transitions. - return; - } - timers.deleteTimer(timestamp, timeDomain); - } - - @Override - public Instant currentProcessingTime() { - return timers.currentProcessingTime(); - } - - @Override - @Nullable - public Instant currentSynchronizedProcessingTime() { - return timers.currentSynchronizedProcessingTime(); - } - - @Override - public Instant currentEventTime() { - return timers.currentEventTime(); - } - } - - private class MergingTriggerInfoImpl - extends TriggerInfoImpl implements Trigger.MergingTriggerInfo { - - private final Map finishedSets; - - public MergingTriggerInfoImpl( - ExecutableTrigger trigger, - FinishedTriggers finishedSet, - Trigger.TriggerContext context, - Map finishedSets) { - super(trigger, finishedSet, context); - this.finishedSets = finishedSets; - } - - @Override - public boolean finishedInAnyMergingWindow() { - for (FinishedTriggers finishedSet : finishedSets.values()) { - if (finishedSet.isFinished(trigger)) { - return true; - } - } - return false; - } - - @Override - public boolean finishedInAllMergingWindows() { - for (FinishedTriggers finishedSet : finishedSets.values()) { - if (!finishedSet.isFinished(trigger)) { - return false; - } - } - return true; - } - } - - private class StateAccessorImpl implements StateAccessor { - protected final int triggerIndex; - protected final StateNamespace windowNamespace; - - public StateAccessorImpl( - W window, - ExecutableTrigger trigger) { - this.triggerIndex = trigger.getTriggerIndex(); - this.windowNamespace = namespaceFor(window); - } - - protected StateNamespace namespaceFor(W window) { - return StateNamespaces.windowAndTrigger(windowCoder, window, triggerIndex); - } - - @Override - public StateT access(StateTag address) { - return stateInternals.state(windowNamespace, address); - } - } - - private class MergingStateAccessorImpl extends StateAccessorImpl - implements MergingStateAccessor { - private final Collection activeToBeMerged; - - public MergingStateAccessorImpl(ExecutableTrigger trigger, Collection activeToBeMerged, - W mergeResult) { - super(mergeResult, trigger); - this.activeToBeMerged = activeToBeMerged; - } - - @Override - public StateT access( - StateTag address) { - return stateInternals.state(windowNamespace, address); - } - - @Override - public Map accessInEachMergingWindow( - StateTag address) { - ImmutableMap.Builder builder = ImmutableMap.builder(); - for (W mergingWindow : activeToBeMerged) { - StateT stateForWindow = stateInternals.state(namespaceFor(mergingWindow), address); - builder.put(mergingWindow, stateForWindow); - } - return builder.build(); - } - } - - private class TriggerContextImpl extends Trigger.TriggerContext { - - private final W window; - private final StateAccessorImpl state; - private final Timers timers; - private final TriggerInfoImpl triggerInfo; - - private TriggerContextImpl( - W window, - Timers timers, - ExecutableTrigger trigger, - FinishedTriggers finishedSet) { - trigger.getSpec().super(); - this.window = window; - this.state = new StateAccessorImpl(window, trigger); - this.timers = new TriggerTimers(window, timers); - this.triggerInfo = new TriggerInfoImpl(trigger, finishedSet, this); - } - - @Override - public Trigger.TriggerContext forTrigger(ExecutableTrigger trigger) { - return new TriggerContextImpl(window, timers, trigger, triggerInfo.finishedSet); - } - - @Override - public TriggerInfo trigger() { - return triggerInfo; - } - - @Override - public StateAccessor state() { - return state; - } - - @Override - public W window() { - return window; - } - - @Override - public void deleteTimer(Instant timestamp, TimeDomain domain) { - timers.deleteTimer(timestamp, domain); - } - - @Override - public Instant currentProcessingTime() { - return timers.currentProcessingTime(); - } - - @Override - @Nullable - public Instant currentSynchronizedProcessingTime() { - return timers.currentSynchronizedProcessingTime(); - } - - @Override - @Nullable - public Instant currentEventTime() { - return timers.currentEventTime(); - } - } - - private class OnElementContextImpl extends Trigger.OnElementContext { - - private final W window; - private final StateAccessorImpl state; - private final Timers timers; - private final TriggerInfoImpl triggerInfo; - private final Instant eventTimestamp; - - private OnElementContextImpl( - W window, - Timers timers, - ExecutableTrigger trigger, - FinishedTriggers finishedSet, - Instant eventTimestamp) { - trigger.getSpec().super(); - this.window = window; - this.state = new StateAccessorImpl(window, trigger); - this.timers = new TriggerTimers(window, timers); - this.triggerInfo = new TriggerInfoImpl(trigger, finishedSet, this); - this.eventTimestamp = eventTimestamp; - } - - - @Override - public Instant eventTimestamp() { - return eventTimestamp; - } - - @Override - public Trigger.OnElementContext forTrigger(ExecutableTrigger trigger) { - return new OnElementContextImpl( - window, timers, trigger, triggerInfo.finishedSet, eventTimestamp); - } - - @Override - public TriggerInfo trigger() { - return triggerInfo; - } - - @Override - public StateAccessor state() { - return state; - } - - @Override - public W window() { - return window; - } - - @Override - public void setTimer(Instant timestamp, TimeDomain domain) { - timers.setTimer(timestamp, domain); - } - - - @Override - public void deleteTimer(Instant timestamp, TimeDomain domain) { - timers.deleteTimer(timestamp, domain); - } - - @Override - public Instant currentProcessingTime() { - return timers.currentProcessingTime(); - } - - @Override - @Nullable - public Instant currentSynchronizedProcessingTime() { - return timers.currentSynchronizedProcessingTime(); - } - - @Override - @Nullable - public Instant currentEventTime() { - return timers.currentEventTime(); - } - } - - private class OnMergeContextImpl extends Trigger.OnMergeContext { - private final MergingStateAccessor state; - private final W window; - private final Collection mergingWindows; - private final Timers timers; - private final MergingTriggerInfoImpl triggerInfo; - - private OnMergeContextImpl( - W window, - Timers timers, - ExecutableTrigger trigger, - FinishedTriggers finishedSet, - Map finishedSets) { - trigger.getSpec().super(); - this.mergingWindows = finishedSets.keySet(); - this.window = window; - this.state = new MergingStateAccessorImpl(trigger, mergingWindows, window); - this.timers = new TriggerTimers(window, timers); - this.triggerInfo = new MergingTriggerInfoImpl(trigger, finishedSet, this, finishedSets); - } - - @Override - public Trigger.OnMergeContext forTrigger(ExecutableTrigger trigger) { - return new OnMergeContextImpl( - window, timers, trigger, triggerInfo.finishedSet, triggerInfo.finishedSets); - } - - @Override - public MergingStateAccessor state() { - return state; - } - - @Override - public MergingTriggerInfo trigger() { - return triggerInfo; - } - - @Override - public W window() { - return window; - } - - @Override - public void setTimer(Instant timestamp, TimeDomain domain) { - timers.setTimer(timestamp, domain); - } - - @Override - public void deleteTimer(Instant timestamp, TimeDomain domain) { - timers.setTimer(timestamp, domain); - - } - - @Override - public Instant currentProcessingTime() { - return timers.currentProcessingTime(); - } - - @Override - @Nullable - public Instant currentSynchronizedProcessingTime() { - return timers.currentSynchronizedProcessingTime(); - } - - @Override - @Nullable - public Instant currentEventTime() { - return timers.currentEventTime(); - } - } -} diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/windowing/AfterAllTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/windowing/AfterAllTest.java index b5912296bd84..5b213c763350 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/windowing/AfterAllTest.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/windowing/AfterAllTest.java @@ -18,13 +18,8 @@ package org.apache.beam.sdk.transforms.windowing; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; import org.apache.beam.sdk.transforms.windowing.Trigger.OnceTrigger; -import org.apache.beam.sdk.util.TriggerTester; -import org.apache.beam.sdk.util.TriggerTester.SimpleTriggerTester; -import org.joda.time.Duration; import org.joda.time.Instant; import org.junit.Test; import org.junit.runner.RunWith; @@ -36,99 +31,6 @@ @RunWith(JUnit4.class) public class AfterAllTest { - private SimpleTriggerTester tester; - - @Test - public void testT1FiresFirst() throws Exception { - tester = TriggerTester.forTrigger( - AfterAll.of( - AfterPane.elementCountAtLeast(1), - AfterPane.elementCountAtLeast(2)), - FixedWindows.of(Duration.millis(100))); - - IntervalWindow window = new IntervalWindow(new Instant(0), new Instant(100)); - - tester.injectElements(1); - assertFalse(tester.shouldFire(window)); - - tester.injectElements(2); - assertTrue(tester.shouldFire(window)); - tester.fireIfShouldFire(window); - assertTrue(tester.isMarkedFinished(window)); - } - - @Test - public void testT2FiresFirst() throws Exception { - tester = TriggerTester.forTrigger( - AfterAll.of( - AfterPane.elementCountAtLeast(2), - AfterPane.elementCountAtLeast(1)), - FixedWindows.of(Duration.millis(100))); - - IntervalWindow window = new IntervalWindow(new Instant(0), new Instant(100)); - - tester.injectElements(1); - assertFalse(tester.shouldFire(window)); - - tester.injectElements(2); - assertTrue(tester.shouldFire(window)); - tester.fireIfShouldFire(window); - assertTrue(tester.isMarkedFinished(window)); - } - - /** - * Tests that the AfterAll properly unsets finished bits when a merge causing it to become - * unfinished. - */ - @Test - public void testOnMergeRewinds() throws Exception { - tester = TriggerTester.forTrigger( - AfterEach.inOrder( - AfterAll.of( - AfterWatermark.pastEndOfWindow(), - AfterPane.elementCountAtLeast(1)), - Repeatedly.forever(AfterPane.elementCountAtLeast(1))), - Sessions.withGapDuration(Duration.millis(10))); - - tester.injectElements(1); - IntervalWindow firstWindow = new IntervalWindow(new Instant(1), new Instant(11)); - - tester.injectElements(5); - IntervalWindow secondWindow = new IntervalWindow(new Instant(5), new Instant(15)); - - // Finish the AfterAll in the first window - tester.advanceInputWatermark(new Instant(11)); - assertTrue(tester.shouldFire(firstWindow)); - assertFalse(tester.shouldFire(secondWindow)); - tester.fireIfShouldFire(firstWindow); - - // Merge them; the AfterAll should not be finished - tester.mergeWindows(); - IntervalWindow mergedWindow = new IntervalWindow(new Instant(1), new Instant(15)); - assertFalse(tester.isMarkedFinished(mergedWindow)); - - // Confirm that we are back on the first trigger by probing that it is not ready to fire - // after an element (with merging) - tester.injectElements(3); - tester.mergeWindows(); - assertFalse(tester.shouldFire(mergedWindow)); - - // Fire the AfterAll in the merged window - tester.advanceInputWatermark(new Instant(15)); - assertTrue(tester.shouldFire(mergedWindow)); - tester.fireIfShouldFire(mergedWindow); - - // Confirm that we are on the second trigger by probing - tester.injectElements(2); - tester.mergeWindows(); - assertTrue(tester.shouldFire(mergedWindow)); - tester.fireIfShouldFire(mergedWindow); - tester.injectElements(2); - tester.mergeWindows(); - assertTrue(tester.shouldFire(mergedWindow)); - tester.fireIfShouldFire(mergedWindow); - } - @Test public void testFireDeadline() throws Exception { BoundedWindow window = new IntervalWindow(new Instant(0), new Instant(10)); diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/windowing/AfterEachTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/windowing/AfterEachTest.java index c413c6ed25c4..00d25e0b30e5 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/windowing/AfterEachTest.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/windowing/AfterEachTest.java @@ -18,19 +18,12 @@ package org.apache.beam.sdk.transforms.windowing; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; import org.apache.beam.sdk.transforms.windowing.Trigger.OnceTrigger; -import org.apache.beam.sdk.util.TriggerTester; -import org.apache.beam.sdk.util.TriggerTester.SimpleTriggerTester; -import org.joda.time.Duration; import org.joda.time.Instant; -import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; -import org.mockito.MockitoAnnotations; /** * Tests for {@link AfterEach}. @@ -38,63 +31,6 @@ @RunWith(JUnit4.class) public class AfterEachTest { - private SimpleTriggerTester tester; - - @Before - public void initMocks() { - MockitoAnnotations.initMocks(this); - } - - /** - * Tests that the {@link AfterEach} trigger fires and finishes the first trigger then the second. - */ - @Test - public void testAfterEachInSequence() throws Exception { - tester = TriggerTester.forTrigger( - AfterEach.inOrder( - Repeatedly.forever(AfterPane.elementCountAtLeast(2)) - .orFinally(AfterPane.elementCountAtLeast(3)), - Repeatedly.forever(AfterPane.elementCountAtLeast(5)) - .orFinally(AfterWatermark.pastEndOfWindow())), - FixedWindows.of(Duration.millis(10))); - - IntervalWindow window = new IntervalWindow(new Instant(0), new Instant(10)); - - // AfterCount(2) not ready - tester.injectElements(1); - assertFalse(tester.shouldFire(window)); - - // AfterCount(2) ready, not finished - tester.injectElements(2); - assertTrue(tester.shouldFire(window)); - tester.fireIfShouldFire(window); - assertFalse(tester.isMarkedFinished(window)); - - // orFinally(AfterCount(3)) ready and will finish the first - tester.injectElements(1, 2, 3); - assertTrue(tester.shouldFire(window)); - tester.fireIfShouldFire(window); - assertFalse(tester.isMarkedFinished(window)); - - // Now running as the second trigger - assertFalse(tester.shouldFire(window)); - // This quantity of elements would fire and finish if it were erroneously still the first - tester.injectElements(1, 2, 3, 4); - assertFalse(tester.shouldFire(window)); - - // Now fire once - tester.injectElements(5); - assertTrue(tester.shouldFire(window)); - tester.fireIfShouldFire(window); - assertFalse(tester.isMarkedFinished(window)); - - // This time advance the watermark to finish the whole mess. - tester.advanceInputWatermark(new Instant(10)); - assertTrue(tester.shouldFire(window)); - tester.fireIfShouldFire(window); - assertTrue(tester.isMarkedFinished(window)); - } - @Test public void testFireDeadline() throws Exception { BoundedWindow window = new IntervalWindow(new Instant(0), new Instant(10)); diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/windowing/AfterFirstTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/windowing/AfterFirstTest.java index 415060b6c22b..2887edbadd2e 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/windowing/AfterFirstTest.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/windowing/AfterFirstTest.java @@ -18,22 +18,12 @@ package org.apache.beam.sdk.transforms.windowing; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; -import static org.mockito.Mockito.when; import org.apache.beam.sdk.transforms.windowing.Trigger.OnceTrigger; -import org.apache.beam.sdk.util.TriggerTester; -import org.apache.beam.sdk.util.TriggerTester.SimpleTriggerTester; -import org.joda.time.Duration; import org.joda.time.Instant; -import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; -import org.mockito.Mock; -import org.mockito.Mockito; -import org.mockito.MockitoAnnotations; /** * Tests for {@link AfterFirst}. @@ -41,116 +31,6 @@ @RunWith(JUnit4.class) public class AfterFirstTest { - @Mock private OnceTrigger mockTrigger1; - @Mock private OnceTrigger mockTrigger2; - private SimpleTriggerTester tester; - private static Trigger.TriggerContext anyTriggerContext() { - return Mockito.any(); - } - - @Before - public void initMocks() { - MockitoAnnotations.initMocks(this); - } - - @Test - public void testNeitherShouldFireFixedWindows() throws Exception { - tester = TriggerTester.forTrigger( - AfterFirst.of(mockTrigger1, mockTrigger2), FixedWindows.of(Duration.millis(10))); - - tester.injectElements(1); - IntervalWindow window = new IntervalWindow(new Instant(0), new Instant(10)); - - when(mockTrigger1.shouldFire(anyTriggerContext())).thenReturn(false); - when(mockTrigger2.shouldFire(anyTriggerContext())).thenReturn(false); - - assertFalse(tester.shouldFire(window)); // should not fire - assertFalse(tester.isMarkedFinished(window)); // not finished - } - - @Test - public void testOnlyT1ShouldFireFixedWindows() throws Exception { - tester = TriggerTester.forTrigger( - AfterFirst.of(mockTrigger1, mockTrigger2), FixedWindows.of(Duration.millis(10))); - tester.injectElements(1); - IntervalWindow window = new IntervalWindow(new Instant(1), new Instant(11)); - - when(mockTrigger1.shouldFire(anyTriggerContext())).thenReturn(true); - when(mockTrigger2.shouldFire(anyTriggerContext())).thenReturn(false); - - assertTrue(tester.shouldFire(window)); // should fire - - tester.fireIfShouldFire(window); - assertTrue(tester.isMarkedFinished(window)); - } - - @Test - public void testOnlyT2ShouldFireFixedWindows() throws Exception { - tester = TriggerTester.forTrigger( - AfterFirst.of(mockTrigger1, mockTrigger2), FixedWindows.of(Duration.millis(10))); - tester.injectElements(1); - IntervalWindow window = new IntervalWindow(new Instant(1), new Instant(11)); - - when(mockTrigger1.shouldFire(anyTriggerContext())).thenReturn(false); - when(mockTrigger2.shouldFire(anyTriggerContext())).thenReturn(true); - assertTrue(tester.shouldFire(window)); // should fire - - tester.fireIfShouldFire(window); // now finished - assertTrue(tester.isMarkedFinished(window)); - } - - @Test - public void testBothShouldFireFixedWindows() throws Exception { - tester = TriggerTester.forTrigger( - AfterFirst.of(mockTrigger1, mockTrigger2), FixedWindows.of(Duration.millis(10))); - tester.injectElements(1); - IntervalWindow window = new IntervalWindow(new Instant(1), new Instant(11)); - - when(mockTrigger1.shouldFire(anyTriggerContext())).thenReturn(true); - when(mockTrigger2.shouldFire(anyTriggerContext())).thenReturn(true); - assertTrue(tester.shouldFire(window)); // should fire - - tester.fireIfShouldFire(window); - assertTrue(tester.isMarkedFinished(window)); - } - - /** - * Tests that if the first trigger rewinds to be non-finished in the merged window, - * then it becomes the currently active trigger again, with real triggers. - */ - @Test - public void testShouldFireAfterMerge() throws Exception { - tester = TriggerTester.forTrigger( - AfterEach.inOrder( - AfterFirst.of(AfterPane.elementCountAtLeast(5), - AfterWatermark.pastEndOfWindow()), - Repeatedly.forever(AfterPane.elementCountAtLeast(1))), - Sessions.withGapDuration(Duration.millis(10))); - - // Finished the AfterFirst in the first window - tester.injectElements(1); - IntervalWindow firstWindow = new IntervalWindow(new Instant(1), new Instant(11)); - assertFalse(tester.shouldFire(firstWindow)); - tester.advanceInputWatermark(new Instant(11)); - assertTrue(tester.shouldFire(firstWindow)); - tester.fireIfShouldFire(firstWindow); - - // Set up second window where it is not done - tester.injectElements(5); - IntervalWindow secondWindow = new IntervalWindow(new Instant(5), new Instant(15)); - assertFalse(tester.shouldFire(secondWindow)); - - // Merge them, if the merged window were on the second trigger, it would be ready - tester.mergeWindows(); - IntervalWindow mergedWindow = new IntervalWindow(new Instant(1), new Instant(15)); - assertFalse(tester.shouldFire(mergedWindow)); - - // Now adding 3 more makes the AfterFirst ready to fire - tester.injectElements(1, 2, 3, 4, 5); - tester.mergeWindows(); - assertTrue(tester.shouldFire(mergedWindow)); - } - @Test public void testFireDeadline() throws Exception { BoundedWindow window = new IntervalWindow(new Instant(0), new Instant(10)); diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/windowing/AfterPaneTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/windowing/AfterPaneTest.java index 38d030ec6be1..1bff80a37e4f 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/windowing/AfterPaneTest.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/windowing/AfterPaneTest.java @@ -18,12 +18,7 @@ package org.apache.beam.sdk.transforms.windowing; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; -import org.apache.beam.sdk.util.TriggerTester; -import org.apache.beam.sdk.util.TriggerTester.SimpleTriggerTester; -import org.joda.time.Duration; import org.joda.time.Instant; import org.junit.Test; import org.junit.runner.RunWith; @@ -35,78 +30,6 @@ @RunWith(JUnit4.class) public class AfterPaneTest { - SimpleTriggerTester tester; - /** - * Tests that the trigger does fire when enough elements are in a window, and that it only - * fires that window (no leakage). - */ - @Test - public void testAfterPaneElementCountFixedWindows() throws Exception { - tester = TriggerTester.forTrigger( - AfterPane.elementCountAtLeast(2), - FixedWindows.of(Duration.millis(10))); - - tester.injectElements(1); // [0, 10) - IntervalWindow window = new IntervalWindow(new Instant(0), new Instant(10)); - assertFalse(tester.shouldFire(window)); - - tester.injectElements(2); // [0, 10) - tester.injectElements(11); // [10, 20) - - assertTrue(tester.shouldFire(window)); // ready to fire - tester.fireIfShouldFire(window); // and finished - assertTrue(tester.isMarkedFinished(window)); - - // But don't finish the other window - assertFalse(tester.isMarkedFinished(new IntervalWindow(new Instant(10), new Instant(20)))); - } - - @Test - public void testClear() throws Exception { - SimpleTriggerTester tester = TriggerTester.forTrigger( - AfterPane.elementCountAtLeast(2), - FixedWindows.of(Duration.millis(10))); - - tester.injectElements(1, 2, 3); - IntervalWindow window = new IntervalWindow(new Instant(0), new Instant(10)); - tester.clearState(window); - tester.assertCleared(window); - } - - @Test - public void testAfterPaneElementCountSessions() throws Exception { - tester = TriggerTester.forTrigger( - AfterPane.elementCountAtLeast(2), - Sessions.withGapDuration(Duration.millis(10))); - - tester.injectElements( - 1, // in [1, 11) - 2); // in [2, 12) - - assertFalse(tester.shouldFire(new IntervalWindow(new Instant(1), new Instant(11)))); - assertFalse(tester.shouldFire(new IntervalWindow(new Instant(2), new Instant(12)))); - - tester.mergeWindows(); - - IntervalWindow mergedWindow = new IntervalWindow(new Instant(1), new Instant(12)); - assertTrue(tester.shouldFire(mergedWindow)); - tester.fireIfShouldFire(mergedWindow); - assertTrue(tester.isMarkedFinished(mergedWindow)); - - // Because we closed the previous window, we don't have it around to merge with. So there - // will be a new FIRE_AND_FINISH result. - tester.injectElements( - 7, // in [7, 17) - 9); // in [9, 19) - - tester.mergeWindows(); - - IntervalWindow newMergedWindow = new IntervalWindow(new Instant(7), new Instant(19)); - assertTrue(tester.shouldFire(newMergedWindow)); - tester.fireIfShouldFire(newMergedWindow); - assertTrue(tester.isMarkedFinished(newMergedWindow)); - } - @Test public void testFireDeadline() throws Exception { assertEquals(BoundedWindow.TIMESTAMP_MAX_VALUE, diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/windowing/AfterProcessingTimeTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/windowing/AfterProcessingTimeTest.java index 13a7acf8ca1e..4984d7ce75a1 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/windowing/AfterProcessingTimeTest.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/windowing/AfterProcessingTimeTest.java @@ -18,12 +18,9 @@ package org.apache.beam.sdk.transforms.windowing; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import org.apache.beam.sdk.transforms.windowing.Trigger.OnceTrigger; -import org.apache.beam.sdk.util.TriggerTester; -import org.apache.beam.sdk.util.TriggerTester.SimpleTriggerTester; import org.joda.time.Duration; import org.joda.time.Instant; import org.junit.Test; @@ -36,97 +33,6 @@ @RunWith(JUnit4.class) public class AfterProcessingTimeTest { - /** - * Tests the basic property that the trigger does wait for processing time to be - * far enough advanced. - */ - @Test - public void testAfterProcessingTimeFixedWindows() throws Exception { - Duration windowDuration = Duration.millis(10); - SimpleTriggerTester tester = TriggerTester.forTrigger( - AfterProcessingTime - .pastFirstElementInPane() - .plusDelayOf(Duration.millis(5)), - FixedWindows.of(windowDuration)); - - tester.advanceProcessingTime(new Instant(10)); - - // Timer at 15 - tester.injectElements(1); - IntervalWindow firstWindow = new IntervalWindow(new Instant(0), new Instant(10)); - tester.advanceProcessingTime(new Instant(12)); - assertFalse(tester.shouldFire(firstWindow)); - - // Load up elements in the next window, timer at 17 for them - tester.injectElements(11, 12, 13); - IntervalWindow secondWindow = new IntervalWindow(new Instant(10), new Instant(20)); - assertFalse(tester.shouldFire(secondWindow)); - - // Not quite time to fire - tester.advanceProcessingTime(new Instant(14)); - assertFalse(tester.shouldFire(firstWindow)); - assertFalse(tester.shouldFire(secondWindow)); - - // Timer at 19 for these in the first window; it should be ignored since the 15 will fire first - tester.injectElements(2, 3); - - // Advance past the first timer and fire, finishing the first window - tester.advanceProcessingTime(new Instant(16)); - assertTrue(tester.shouldFire(firstWindow)); - assertFalse(tester.shouldFire(secondWindow)); - tester.fireIfShouldFire(firstWindow); - assertTrue(tester.isMarkedFinished(firstWindow)); - - // The next window fires and finishes now - tester.advanceProcessingTime(new Instant(18)); - assertTrue(tester.shouldFire(secondWindow)); - tester.fireIfShouldFire(secondWindow); - assertTrue(tester.isMarkedFinished(secondWindow)); - } - - /** - * Tests that when windows merge, if the trigger is waiting for "N millis after the first - * element" that it is relative to the earlier of the two merged windows. - */ - @Test - public void testClear() throws Exception { - SimpleTriggerTester tester = TriggerTester.forTrigger( - AfterProcessingTime - .pastFirstElementInPane() - .plusDelayOf(Duration.millis(5)), - FixedWindows.of(Duration.millis(10))); - - tester.injectElements(1, 2, 3); - IntervalWindow window = new IntervalWindow(new Instant(0), new Instant(10)); - tester.clearState(window); - tester.assertCleared(window); - } - - @Test - public void testAfterProcessingTimeWithMergingWindow() throws Exception { - SimpleTriggerTester tester = TriggerTester.forTrigger( - AfterProcessingTime - .pastFirstElementInPane() - .plusDelayOf(Duration.millis(5)), - Sessions.withGapDuration(Duration.millis(10))); - - tester.advanceProcessingTime(new Instant(10)); - tester.injectElements(1); // in [1, 11), timer for 15 - IntervalWindow firstWindow = new IntervalWindow(new Instant(1), new Instant(11)); - assertFalse(tester.shouldFire(firstWindow)); - - tester.advanceProcessingTime(new Instant(12)); - tester.injectElements(3); // in [3, 13), timer for 17 - IntervalWindow secondWindow = new IntervalWindow(new Instant(3), new Instant(13)); - assertFalse(tester.shouldFire(secondWindow)); - - tester.mergeWindows(); - IntervalWindow mergedWindow = new IntervalWindow(new Instant(1), new Instant(13)); - - tester.advanceProcessingTime(new Instant(16)); - assertTrue(tester.shouldFire(mergedWindow)); - } - @Test public void testFireDeadline() throws Exception { assertEquals(BoundedWindow.TIMESTAMP_MAX_VALUE, diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/windowing/AfterSynchronizedProcessingTimeTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/windowing/AfterSynchronizedProcessingTimeTest.java index 7e6e938f3c8b..49d44c55dfe4 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/windowing/AfterSynchronizedProcessingTimeTest.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/windowing/AfterSynchronizedProcessingTimeTest.java @@ -18,12 +18,7 @@ package org.apache.beam.sdk.transforms.windowing; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; -import org.apache.beam.sdk.util.TriggerTester; -import org.apache.beam.sdk.util.TriggerTester.SimpleTriggerTester; -import org.joda.time.Duration; import org.joda.time.Instant; import org.junit.Test; import org.junit.runner.RunWith; @@ -37,76 +32,6 @@ public class AfterSynchronizedProcessingTimeTest { private Trigger underTest = new AfterSynchronizedProcessingTime(); - @Test - public void testAfterProcessingTimeWithFixedWindows() throws Exception { - Duration windowDuration = Duration.millis(10); - SimpleTriggerTester tester = TriggerTester.forTrigger( - AfterProcessingTime - .pastFirstElementInPane() - .plusDelayOf(Duration.millis(5)), - FixedWindows.of(windowDuration)); - - tester.advanceProcessingTime(new Instant(10)); - - // Timer at 15 - tester.injectElements(1); - IntervalWindow firstWindow = new IntervalWindow(new Instant(0), new Instant(10)); - tester.advanceProcessingTime(new Instant(12)); - assertFalse(tester.shouldFire(firstWindow)); - - // Load up elements in the next window, timer at 17 for them - tester.injectElements(11, 12, 13); - IntervalWindow secondWindow = new IntervalWindow(new Instant(10), new Instant(20)); - assertFalse(tester.shouldFire(secondWindow)); - - // Not quite time to fire - tester.advanceProcessingTime(new Instant(14)); - assertFalse(tester.shouldFire(firstWindow)); - assertFalse(tester.shouldFire(secondWindow)); - - // Timer at 19 for these in the first window; it should be ignored since the 15 will fire first - tester.injectElements(2, 3); - - // Advance past the first timer and fire, finishing the first window - tester.advanceProcessingTime(new Instant(16)); - assertTrue(tester.shouldFire(firstWindow)); - assertFalse(tester.shouldFire(secondWindow)); - tester.fireIfShouldFire(firstWindow); - assertTrue(tester.isMarkedFinished(firstWindow)); - - // The next window fires and finishes now - tester.advanceProcessingTime(new Instant(18)); - assertTrue(tester.shouldFire(secondWindow)); - tester.fireIfShouldFire(secondWindow); - assertTrue(tester.isMarkedFinished(secondWindow)); - } - - @Test - public void testAfterProcessingTimeWithMergingWindow() throws Exception { - Duration windowDuration = Duration.millis(10); - SimpleTriggerTester tester = TriggerTester.forTrigger( - AfterProcessingTime - .pastFirstElementInPane() - .plusDelayOf(Duration.millis(5)), - Sessions.withGapDuration(windowDuration)); - - tester.advanceProcessingTime(new Instant(10)); - tester.injectElements(1); // in [1, 11), timer for 15 - IntervalWindow firstWindow = new IntervalWindow(new Instant(1), new Instant(11)); - assertFalse(tester.shouldFire(firstWindow)); - - tester.advanceProcessingTime(new Instant(12)); - tester.injectElements(3); // in [3, 13), timer for 17 - IntervalWindow secondWindow = new IntervalWindow(new Instant(3), new Instant(13)); - assertFalse(tester.shouldFire(secondWindow)); - - tester.mergeWindows(); - IntervalWindow mergedWindow = new IntervalWindow(new Instant(1), new Instant(13)); - - tester.advanceProcessingTime(new Instant(16)); - assertTrue(tester.shouldFire(mergedWindow)); - } - @Test public void testFireDeadline() throws Exception { assertEquals(BoundedWindow.TIMESTAMP_MAX_VALUE, diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/windowing/AfterWatermarkTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/windowing/AfterWatermarkTest.java index 084027b3e5b1..a418d63dc550 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/windowing/AfterWatermarkTest.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/windowing/AfterWatermarkTest.java @@ -18,23 +18,10 @@ package org.apache.beam.sdk.transforms.windowing; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; -import static org.mockito.Mockito.doNothing; -import static org.mockito.Mockito.when; -import org.apache.beam.sdk.transforms.windowing.Trigger.OnceTrigger; -import org.apache.beam.sdk.util.TriggerTester; -import org.apache.beam.sdk.util.TriggerTester.SimpleTriggerTester; -import org.joda.time.Duration; -import org.joda.time.Instant; -import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; -import org.mockito.Mock; -import org.mockito.Mockito; -import org.mockito.MockitoAnnotations; /** * Tests the {@link AfterWatermark} triggers. @@ -42,301 +29,6 @@ @RunWith(JUnit4.class) public class AfterWatermarkTest { - @Mock private OnceTrigger mockEarly; - @Mock private OnceTrigger mockLate; - - private SimpleTriggerTester tester; - private static Trigger.TriggerContext anyTriggerContext() { - return Mockito.any(); - } - private static Trigger.OnElementContext anyElementContext() { - return Mockito.any(); - } - - private void injectElements(int... elements) throws Exception { - for (int element : elements) { - doNothing().when(mockEarly).onElement(anyElementContext()); - doNothing().when(mockLate).onElement(anyElementContext()); - tester.injectElements(element); - } - } - - @Before - public void setUp() { - MockitoAnnotations.initMocks(this); - } - - public void testRunningAsTrigger(OnceTrigger mockTrigger, IntervalWindow window) - throws Exception { - - // Don't fire due to mock saying no - when(mockTrigger.shouldFire(anyTriggerContext())).thenReturn(false); - assertFalse(tester.shouldFire(window)); // not ready - - // Fire due to mock trigger; early trigger is required to be a OnceTrigger - when(mockTrigger.shouldFire(anyTriggerContext())).thenReturn(true); - assertTrue(tester.shouldFire(window)); // ready - tester.fireIfShouldFire(window); - assertFalse(tester.isMarkedFinished(window)); - } - - @Test - public void testEarlyAndAtWatermark() throws Exception { - tester = TriggerTester.forTrigger( - AfterWatermark.pastEndOfWindow() - .withEarlyFirings(mockEarly), - FixedWindows.of(Duration.millis(100))); - - injectElements(1); - IntervalWindow window = new IntervalWindow(new Instant(0), new Instant(100)); - - testRunningAsTrigger(mockEarly, window); - - // Fire due to watermark - when(mockEarly.shouldFire(anyTriggerContext())).thenReturn(false); - tester.advanceInputWatermark(new Instant(100)); - assertTrue(tester.shouldFire(window)); - tester.fireIfShouldFire(window); - assertTrue(tester.isMarkedFinished(window)); - } - - @Test - public void testAtWatermarkAndLate() throws Exception { - tester = TriggerTester.forTrigger( - AfterWatermark.pastEndOfWindow() - .withLateFirings(mockLate), - FixedWindows.of(Duration.millis(100))); - - injectElements(1); - IntervalWindow window = new IntervalWindow(new Instant(0), new Instant(100)); - - // No early firing, just double checking - when(mockEarly.shouldFire(anyTriggerContext())).thenReturn(true); - assertFalse(tester.shouldFire(window)); - tester.fireIfShouldFire(window); - assertFalse(tester.isMarkedFinished(window)); - - // Fire due to watermark - when(mockEarly.shouldFire(anyTriggerContext())).thenReturn(false); - tester.advanceInputWatermark(new Instant(100)); - assertTrue(tester.shouldFire(window)); - tester.fireIfShouldFire(window); - assertFalse(tester.isMarkedFinished(window)); - - testRunningAsTrigger(mockLate, window); - } - - @Test - public void testEarlyAndAtWatermarkAndLate() throws Exception { - tester = TriggerTester.forTrigger( - AfterWatermark.pastEndOfWindow() - .withEarlyFirings(mockEarly) - .withLateFirings(mockLate), - FixedWindows.of(Duration.millis(100))); - - injectElements(1); - IntervalWindow window = new IntervalWindow(new Instant(0), new Instant(100)); - - testRunningAsTrigger(mockEarly, window); - - // Fire due to watermark - when(mockEarly.shouldFire(anyTriggerContext())).thenReturn(false); - tester.advanceInputWatermark(new Instant(100)); - assertTrue(tester.shouldFire(window)); - tester.fireIfShouldFire(window); - assertFalse(tester.isMarkedFinished(window)); - - testRunningAsTrigger(mockLate, window); - } - - /** - * Tests that if the EOW is finished in both as well as the merged window, then - * it is finished in the merged result. - * - *

Because windows are discarded when a trigger finishes, we need to embed this - * in a sequence in order to check that it is re-activated. So this test is potentially - * sensitive to other triggers' correctness. - */ - @Test - public void testOnMergeAlreadyFinished() throws Exception { - tester = TriggerTester.forTrigger( - AfterEach.inOrder( - AfterWatermark.pastEndOfWindow(), - Repeatedly.forever(AfterPane.elementCountAtLeast(1))), - Sessions.withGapDuration(Duration.millis(10))); - - tester.injectElements(1); - tester.injectElements(5); - IntervalWindow firstWindow = new IntervalWindow(new Instant(1), new Instant(11)); - IntervalWindow secondWindow = new IntervalWindow(new Instant(5), new Instant(15)); - IntervalWindow mergedWindow = new IntervalWindow(new Instant(1), new Instant(15)); - - // Finish the AfterWatermark.pastEndOfWindow() trigger in both windows - tester.advanceInputWatermark(new Instant(15)); - assertTrue(tester.shouldFire(firstWindow)); - assertTrue(tester.shouldFire(secondWindow)); - tester.fireIfShouldFire(firstWindow); - tester.fireIfShouldFire(secondWindow); - - // Confirm that we are on the second trigger by probing - assertFalse(tester.shouldFire(firstWindow)); - assertFalse(tester.shouldFire(secondWindow)); - tester.injectElements(1); - tester.injectElements(5); - assertTrue(tester.shouldFire(firstWindow)); - assertTrue(tester.shouldFire(secondWindow)); - tester.fireIfShouldFire(firstWindow); - tester.fireIfShouldFire(secondWindow); - - // Merging should leave it finished - tester.mergeWindows(); - - // Confirm that we are on the second trigger by probing - assertFalse(tester.shouldFire(mergedWindow)); - tester.injectElements(1); - assertTrue(tester.shouldFire(mergedWindow)); - } - - /** - * Tests that the trigger rewinds to be non-finished in the merged window. - * - *

Because windows are discarded when a trigger finishes, we need to embed this - * in a sequence in order to check that it is re-activated. So this test is potentially - * sensitive to other triggers' correctness. - */ - @Test - public void testOnMergeRewinds() throws Exception { - tester = TriggerTester.forTrigger( - AfterEach.inOrder( - AfterWatermark.pastEndOfWindow(), - Repeatedly.forever(AfterPane.elementCountAtLeast(1))), - Sessions.withGapDuration(Duration.millis(10))); - - tester.injectElements(1); - tester.injectElements(5); - IntervalWindow firstWindow = new IntervalWindow(new Instant(1), new Instant(11)); - IntervalWindow secondWindow = new IntervalWindow(new Instant(5), new Instant(15)); - IntervalWindow mergedWindow = new IntervalWindow(new Instant(1), new Instant(15)); - - // Finish the AfterWatermark.pastEndOfWindow() trigger in only the first window - tester.advanceInputWatermark(new Instant(11)); - assertTrue(tester.shouldFire(firstWindow)); - assertFalse(tester.shouldFire(secondWindow)); - tester.fireIfShouldFire(firstWindow); - - // Confirm that we are on the second trigger by probing - assertFalse(tester.shouldFire(firstWindow)); - tester.injectElements(1); - assertTrue(tester.shouldFire(firstWindow)); - tester.fireIfShouldFire(firstWindow); - - // Merging should re-activate the watermark trigger in the merged window - tester.mergeWindows(); - - // Confirm that we are not on the second trigger by probing - assertFalse(tester.shouldFire(mergedWindow)); - tester.injectElements(1); - assertFalse(tester.shouldFire(mergedWindow)); - - // And confirm that advancing the watermark fires again - tester.advanceInputWatermark(new Instant(15)); - assertTrue(tester.shouldFire(mergedWindow)); - } - - /** - * Tests that if the EOW is finished in both as well as the merged window, then - * it is finished in the merged result. - * - *

Because windows are discarded when a trigger finishes, we need to embed this - * in a sequence in order to check that it is re-activated. So this test is potentially - * sensitive to other triggers' correctness. - */ - @Test - public void testEarlyAndLateOnMergeAlreadyFinished() throws Exception { - tester = TriggerTester.forTrigger( - AfterWatermark.pastEndOfWindow() - .withEarlyFirings(AfterPane.elementCountAtLeast(100)) - .withLateFirings(AfterPane.elementCountAtLeast(1)), - Sessions.withGapDuration(Duration.millis(10))); - - tester.injectElements(1); - tester.injectElements(5); - IntervalWindow firstWindow = new IntervalWindow(new Instant(1), new Instant(11)); - IntervalWindow secondWindow = new IntervalWindow(new Instant(5), new Instant(15)); - IntervalWindow mergedWindow = new IntervalWindow(new Instant(1), new Instant(15)); - - // Finish the AfterWatermark.pastEndOfWindow() bit of the trigger in both windows - tester.advanceInputWatermark(new Instant(15)); - assertTrue(tester.shouldFire(firstWindow)); - assertTrue(tester.shouldFire(secondWindow)); - tester.fireIfShouldFire(firstWindow); - tester.fireIfShouldFire(secondWindow); - - // Confirm that we are on the late trigger by probing - assertFalse(tester.shouldFire(firstWindow)); - assertFalse(tester.shouldFire(secondWindow)); - tester.injectElements(1); - tester.injectElements(5); - assertTrue(tester.shouldFire(firstWindow)); - assertTrue(tester.shouldFire(secondWindow)); - tester.fireIfShouldFire(firstWindow); - tester.fireIfShouldFire(secondWindow); - - // Merging should leave it on the late trigger - tester.mergeWindows(); - - // Confirm that we are on the late trigger by probing - assertFalse(tester.shouldFire(mergedWindow)); - tester.injectElements(1); - assertTrue(tester.shouldFire(mergedWindow)); - } - - /** - * Tests that the trigger rewinds to be non-finished in the merged window. - * - *

Because windows are discarded when a trigger finishes, we need to embed this - * in a sequence in order to check that it is re-activated. So this test is potentially - * sensitive to other triggers' correctness. - */ - @Test - public void testEarlyAndLateOnMergeRewinds() throws Exception { - tester = TriggerTester.forTrigger( - AfterWatermark.pastEndOfWindow() - .withEarlyFirings(AfterPane.elementCountAtLeast(100)) - .withLateFirings(AfterPane.elementCountAtLeast(1)), - Sessions.withGapDuration(Duration.millis(10))); - - tester.injectElements(1); - tester.injectElements(5); - IntervalWindow firstWindow = new IntervalWindow(new Instant(1), new Instant(11)); - IntervalWindow secondWindow = new IntervalWindow(new Instant(5), new Instant(15)); - IntervalWindow mergedWindow = new IntervalWindow(new Instant(1), new Instant(15)); - - // Finish the AfterWatermark.pastEndOfWindow() bit of the trigger in only the first window - tester.advanceInputWatermark(new Instant(11)); - assertTrue(tester.shouldFire(firstWindow)); - assertFalse(tester.shouldFire(secondWindow)); - tester.fireIfShouldFire(firstWindow); - - // Confirm that we are on the late trigger by probing - assertFalse(tester.shouldFire(firstWindow)); - tester.injectElements(1); - assertTrue(tester.shouldFire(firstWindow)); - tester.fireIfShouldFire(firstWindow); - - // Merging should re-activate the early trigger in the merged window - tester.mergeWindows(); - - // Confirm that we are not on the second trigger by probing - assertFalse(tester.shouldFire(mergedWindow)); - tester.injectElements(1); - assertFalse(tester.shouldFire(mergedWindow)); - - // And confirm that advancing the watermark fires again - tester.advanceInputWatermark(new Instant(15)); - assertTrue(tester.shouldFire(mergedWindow)); - } - @Test public void testFromEndOfWindowToString() { Trigger trigger = AfterWatermark.pastEndOfWindow(); diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/windowing/DefaultTriggerTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/windowing/DefaultTriggerTest.java index 673e5554b008..ee1c44afdbf1 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/windowing/DefaultTriggerTest.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/windowing/DefaultTriggerTest.java @@ -18,12 +18,7 @@ package org.apache.beam.sdk.transforms.windowing; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; -import org.apache.beam.sdk.util.TriggerTester; -import org.apache.beam.sdk.util.TriggerTester.SimpleTriggerTester; -import org.joda.time.Duration; import org.joda.time.Instant; import org.junit.Test; import org.junit.runner.RunWith; @@ -36,131 +31,6 @@ @RunWith(JUnit4.class) public class DefaultTriggerTest { - SimpleTriggerTester tester; - - @Test - public void testDefaultTriggerFixedWindows() throws Exception { - tester = TriggerTester.forTrigger( - DefaultTrigger.of(), - FixedWindows.of(Duration.millis(100))); - - tester.injectElements( - 1, // [0, 100) - 101); // [100, 200) - - IntervalWindow firstWindow = new IntervalWindow(new Instant(0), new Instant(100)); - IntervalWindow secondWindow = new IntervalWindow(new Instant(100), new Instant(200)); - - // Advance the watermark almost to the end of the first window. - tester.advanceInputWatermark(new Instant(99)); - assertFalse(tester.shouldFire(firstWindow)); - assertFalse(tester.shouldFire(secondWindow)); - - // Advance watermark past end of the first window, which is then ready - tester.advanceInputWatermark(new Instant(100)); - assertTrue(tester.shouldFire(firstWindow)); - assertFalse(tester.shouldFire(secondWindow)); - - // Fire, but the first window is still allowed to fire - tester.fireIfShouldFire(firstWindow); - assertTrue(tester.shouldFire(firstWindow)); - assertFalse(tester.shouldFire(secondWindow)); - - // Advance watermark to 200, then both are ready - tester.advanceInputWatermark(new Instant(200)); - assertTrue(tester.shouldFire(firstWindow)); - assertTrue(tester.shouldFire(secondWindow)); - - assertFalse(tester.isMarkedFinished(firstWindow)); - assertFalse(tester.isMarkedFinished(secondWindow)); - } - - @Test - public void testDefaultTriggerSlidingWindows() throws Exception { - tester = TriggerTester.forTrigger( - DefaultTrigger.of(), - SlidingWindows.of(Duration.millis(100)).every(Duration.millis(50))); - - tester.injectElements( - 1, // [-50, 50), [0, 100) - 50); // [0, 100), [50, 150) - - IntervalWindow firstWindow = new IntervalWindow(new Instant(-50), new Instant(50)); - IntervalWindow secondWindow = new IntervalWindow(new Instant(0), new Instant(100)); - IntervalWindow thirdWindow = new IntervalWindow(new Instant(50), new Instant(150)); - - assertFalse(tester.shouldFire(firstWindow)); - assertFalse(tester.shouldFire(secondWindow)); - assertFalse(tester.shouldFire(thirdWindow)); - - // At 50, the first becomes ready; it stays ready after firing - tester.advanceInputWatermark(new Instant(50)); - assertTrue(tester.shouldFire(firstWindow)); - assertFalse(tester.shouldFire(secondWindow)); - assertFalse(tester.shouldFire(thirdWindow)); - tester.fireIfShouldFire(firstWindow); - assertTrue(tester.shouldFire(firstWindow)); - assertFalse(tester.shouldFire(secondWindow)); - assertFalse(tester.shouldFire(thirdWindow)); - - // At 99, the first is still the only one ready - tester.advanceInputWatermark(new Instant(99)); - assertTrue(tester.shouldFire(firstWindow)); - assertFalse(tester.shouldFire(secondWindow)); - assertFalse(tester.shouldFire(thirdWindow)); - - // At 100, the first and second are ready - tester.advanceInputWatermark(new Instant(100)); - assertTrue(tester.shouldFire(firstWindow)); - assertTrue(tester.shouldFire(secondWindow)); - assertFalse(tester.shouldFire(thirdWindow)); - tester.fireIfShouldFire(firstWindow); - - assertFalse(tester.isMarkedFinished(firstWindow)); - assertFalse(tester.isMarkedFinished(secondWindow)); - assertFalse(tester.isMarkedFinished(thirdWindow)); - } - - @Test - public void testDefaultTriggerSessions() throws Exception { - tester = TriggerTester.forTrigger( - DefaultTrigger.of(), - Sessions.withGapDuration(Duration.millis(100))); - - tester.injectElements( - 1, // [1, 101) - 50); // [50, 150) - tester.mergeWindows(); - - IntervalWindow firstWindow = new IntervalWindow(new Instant(1), new Instant(101)); - IntervalWindow secondWindow = new IntervalWindow(new Instant(50), new Instant(150)); - IntervalWindow mergedWindow = new IntervalWindow(new Instant(1), new Instant(150)); - - // Not ready in any window yet - tester.advanceInputWatermark(new Instant(100)); - assertFalse(tester.shouldFire(firstWindow)); - assertFalse(tester.shouldFire(secondWindow)); - assertFalse(tester.shouldFire(mergedWindow)); - - // The first window is "ready": the caller owns knowledge of which windows are merged away - tester.advanceInputWatermark(new Instant(149)); - assertTrue(tester.shouldFire(firstWindow)); - assertFalse(tester.shouldFire(secondWindow)); - assertFalse(tester.shouldFire(mergedWindow)); - - // Now ready on all windows - tester.advanceInputWatermark(new Instant(150)); - assertTrue(tester.shouldFire(firstWindow)); - assertTrue(tester.shouldFire(secondWindow)); - assertTrue(tester.shouldFire(mergedWindow)); - - // Ensure it repeats - tester.fireIfShouldFire(mergedWindow); - assertTrue(tester.shouldFire(mergedWindow)); - - assertFalse(tester.isMarkedFinished(mergedWindow)); - } - @Test public void testFireDeadline() throws Exception { assertEquals(new Instant(9), DefaultTrigger.of().getWatermarkThatGuaranteesFiring( diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/windowing/NeverTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/windowing/NeverTest.java index fb2b4d5c730d..10528739c4f0 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/windowing/NeverTest.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/windowing/NeverTest.java @@ -17,40 +17,26 @@ */ package org.apache.beam.sdk.transforms.windowing; -import static org.hamcrest.Matchers.is; -import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertEquals; -import org.apache.beam.sdk.util.TriggerTester; -import org.apache.beam.sdk.util.TriggerTester.SimpleTriggerTester; -import org.apache.beam.sdk.values.TimestampedValue; -import org.joda.time.Duration; import org.joda.time.Instant; -import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; -/** - * Tests for {@link Never}. - */ +/** Tests for {@link Never}. */ @RunWith(JUnit4.class) public class NeverTest { - private SimpleTriggerTester triggerTester; - - @Before - public void setup() throws Exception { - triggerTester = - TriggerTester.forTrigger( - Never.ever(), FixedWindows.of(Duration.standardMinutes(5))); + @Test + public void testFireDeadline() throws Exception { + assertEquals( + BoundedWindow.TIMESTAMP_MAX_VALUE, + Never.ever() + .getWatermarkThatGuaranteesFiring(new IntervalWindow(new Instant(0), new Instant(10)))); } @Test - public void falseAfterEndOfWindow() throws Exception { - triggerTester.injectElements(TimestampedValue.of(1, new Instant(1))); - IntervalWindow window = - new IntervalWindow(new Instant(0), new Instant(0).plus(Duration.standardMinutes(5))); - assertThat(triggerTester.shouldFire(window), is(false)); - triggerTester.advanceInputWatermark(BoundedWindow.TIMESTAMP_MAX_VALUE); - assertThat(triggerTester.shouldFire(window), is(false)); + public void testContinuation() throws Exception { + assertEquals(Never.ever(), Never.ever().getContinuationTrigger()); } } diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/windowing/OrFinallyTriggerTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/windowing/OrFinallyTriggerTest.java index 7289d97d7b63..6e61e10cffaa 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/windowing/OrFinallyTriggerTest.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/windowing/OrFinallyTriggerTest.java @@ -18,13 +18,8 @@ package org.apache.beam.sdk.transforms.windowing; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; import org.apache.beam.sdk.transforms.windowing.Trigger.OnceTrigger; -import org.apache.beam.sdk.util.TriggerTester; -import org.apache.beam.sdk.util.TriggerTester.SimpleTriggerTester; -import org.joda.time.Duration; import org.joda.time.Instant; import org.junit.Test; import org.junit.runner.RunWith; @@ -36,137 +31,6 @@ @RunWith(JUnit4.class) public class OrFinallyTriggerTest { - private SimpleTriggerTester tester; - - /** - * Tests that for {@code OrFinally(actual, ...)} when {@code actual} - * fires and finishes, the {@code OrFinally} also fires and finishes. - */ - @Test - public void testActualFiresAndFinishes() throws Exception { - tester = TriggerTester.forTrigger( - new OrFinallyTrigger( - AfterPane.elementCountAtLeast(2), - AfterPane.elementCountAtLeast(100)), - FixedWindows.of(Duration.millis(100))); - - IntervalWindow window = new IntervalWindow(new Instant(0), new Instant(100)); - - // Not yet firing - tester.injectElements(1); - assertFalse(tester.shouldFire(window)); - assertFalse(tester.isMarkedFinished(window)); - - // The actual fires and finishes - tester.injectElements(2); - assertTrue(tester.shouldFire(window)); - tester.fireIfShouldFire(window); - assertTrue(tester.isMarkedFinished(window)); - } - - /** - * Tests that for {@code OrFinally(actual, ...)} when {@code actual} - * fires but does not finish, the {@code OrFinally} also fires and also does not - * finish. - */ - @Test - public void testActualFiresOnly() throws Exception { - tester = TriggerTester.forTrigger( - new OrFinallyTrigger( - Repeatedly.forever(AfterPane.elementCountAtLeast(2)), - AfterPane.elementCountAtLeast(100)), - FixedWindows.of(Duration.millis(100))); - - IntervalWindow window = new IntervalWindow(new Instant(0), new Instant(100)); - - // Not yet firing - tester.injectElements(1); - assertFalse(tester.shouldFire(window)); - assertFalse(tester.isMarkedFinished(window)); - - // The actual fires but does not finish - tester.injectElements(2); - assertTrue(tester.shouldFire(window)); - tester.fireIfShouldFire(window); - assertFalse(tester.isMarkedFinished(window)); - - // And again - tester.injectElements(3, 4); - assertTrue(tester.shouldFire(window)); - tester.fireIfShouldFire(window); - assertFalse(tester.isMarkedFinished(window)); - } - - /** - * Tests that if the first trigger rewinds to be non-finished in the merged window, - * then it becomes the currently active trigger again, with real triggers. - */ - @Test - public void testShouldFireAfterMerge() throws Exception { - tester = TriggerTester.forTrigger( - AfterEach.inOrder( - AfterPane.elementCountAtLeast(5) - .orFinally(AfterWatermark.pastEndOfWindow()), - Repeatedly.forever(AfterPane.elementCountAtLeast(1))), - Sessions.withGapDuration(Duration.millis(10))); - - // Finished the orFinally in the first window - tester.injectElements(1); - IntervalWindow firstWindow = new IntervalWindow(new Instant(1), new Instant(11)); - assertFalse(tester.shouldFire(firstWindow)); - tester.advanceInputWatermark(new Instant(11)); - assertTrue(tester.shouldFire(firstWindow)); - tester.fireIfShouldFire(firstWindow); - - // Set up second window where it is not done - tester.injectElements(5); - IntervalWindow secondWindow = new IntervalWindow(new Instant(5), new Instant(15)); - assertFalse(tester.shouldFire(secondWindow)); - - // Merge them, if the merged window were on the second trigger, it would be ready - tester.mergeWindows(); - IntervalWindow mergedWindow = new IntervalWindow(new Instant(1), new Instant(15)); - assertFalse(tester.shouldFire(mergedWindow)); - - // Now adding 3 more makes the main trigger ready to fire - tester.injectElements(1, 2, 3, 4, 5); - tester.mergeWindows(); - assertTrue(tester.shouldFire(mergedWindow)); - } - - /** - * Tests that for {@code OrFinally(actual, until)} when {@code actual} - * fires but does not finish, then {@code until} fires and finishes, the - * whole thing fires and finished. - */ - @Test - public void testActualFiresButUntilFinishes() throws Exception { - tester = TriggerTester.forTrigger( - new OrFinallyTrigger( - Repeatedly.forever(AfterPane.elementCountAtLeast(2)), - AfterPane.elementCountAtLeast(3)), - FixedWindows.of(Duration.millis(10))); - - IntervalWindow window = new IntervalWindow(new Instant(0), new Instant(10)); - - // Before any firing - tester.injectElements(1); - assertFalse(tester.shouldFire(window)); - assertFalse(tester.isMarkedFinished(window)); - - // The actual fires but doesn't finish - tester.injectElements(2); - assertTrue(tester.shouldFire(window)); - tester.fireIfShouldFire(window); - assertFalse(tester.isMarkedFinished(window)); - - // The until fires and finishes; the trigger is finished - tester.injectElements(3); - assertTrue(tester.shouldFire(window)); - tester.fireIfShouldFire(window); - assertTrue(tester.isMarkedFinished(window)); - } - @Test public void testFireDeadline() throws Exception { BoundedWindow window = new IntervalWindow(new Instant(0), new Instant(10)); diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/windowing/RepeatedlyTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/windowing/RepeatedlyTest.java index 6e8930dbd883..55cb77efc3fc 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/windowing/RepeatedlyTest.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/windowing/RepeatedlyTest.java @@ -19,14 +19,9 @@ import static org.hamcrest.Matchers.equalTo; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertThat; -import static org.junit.Assert.assertTrue; -import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; -import org.apache.beam.sdk.util.TriggerTester; -import org.apache.beam.sdk.util.TriggerTester.SimpleTriggerTester; import org.joda.time.Duration; import org.joda.time.Instant; import org.junit.Test; @@ -36,51 +31,17 @@ import org.mockito.Mockito; import org.mockito.MockitoAnnotations; -/** - * Tests for {@link Repeatedly}. - */ +/** Tests for {@link Repeatedly}. */ @RunWith(JUnit4.class) public class RepeatedlyTest { @Mock private Trigger mockTrigger; - private SimpleTriggerTester tester; - private static Trigger.TriggerContext anyTriggerContext() { - return Mockito.any(); - } public void setUp(WindowFn windowFn) throws Exception { MockitoAnnotations.initMocks(this); - tester = TriggerTester.forTrigger(Repeatedly.forever(mockTrigger), windowFn); - } - - /** - * Tests that onElement correctly passes the data on to the subtrigger. - */ - @Test - public void testOnElement() throws Exception { - setUp(FixedWindows.of(Duration.millis(10))); - tester.injectElements(37); - verify(mockTrigger).onElement(Mockito.any()); - } - - /** - * Tests that the repeatedly is ready to fire whenever the subtrigger is ready. - */ - @Test - public void testShouldFire() throws Exception { - setUp(FixedWindows.of(Duration.millis(10))); - - when(mockTrigger.shouldFire(anyTriggerContext())).thenReturn(true); - assertTrue(tester.shouldFire(new IntervalWindow(new Instant(0), new Instant(10)))); - - when(mockTrigger.shouldFire(Mockito.any())) - .thenReturn(false); - assertFalse(tester.shouldFire(new IntervalWindow(new Instant(0), new Instant(10)))); } - /** - * Tests that the watermark that guarantees firing is that of the subtrigger. - */ + /** Tests that the watermark that guarantees firing is that of the subtrigger. */ @Test public void testFireDeadline() throws Exception { setUp(FixedWindows.of(Duration.millis(10))); @@ -106,119 +67,17 @@ public void testContinuation() throws Exception { repeatedly.getContinuationTrigger().getContinuationTrigger()); } - @Test - public void testShouldFireAfterMerge() throws Exception { - tester = TriggerTester.forTrigger( - Repeatedly.forever(AfterPane.elementCountAtLeast(2)), - Sessions.withGapDuration(Duration.millis(10))); - - tester.injectElements(1); - IntervalWindow firstWindow = new IntervalWindow(new Instant(1), new Instant(11)); - assertFalse(tester.shouldFire(firstWindow)); - - tester.injectElements(5); - IntervalWindow secondWindow = new IntervalWindow(new Instant(5), new Instant(15)); - assertFalse(tester.shouldFire(secondWindow)); - - // Merge them, if the merged window were on the second trigger, it would be ready - tester.mergeWindows(); - IntervalWindow mergedWindow = new IntervalWindow(new Instant(1), new Instant(15)); - assertTrue(tester.shouldFire(mergedWindow)); - } - - @Test - public void testRepeatedlyAfterFirstElementCount() throws Exception { - SimpleTriggerTester tester = - TriggerTester.forTrigger( - Repeatedly.forever( - AfterFirst.of( - AfterProcessingTime.pastFirstElementInPane() - .plusDelayOf(Duration.standardMinutes(15)), - AfterPane.elementCountAtLeast(5))), - new GlobalWindows()); - - GlobalWindow window = GlobalWindow.INSTANCE; - - tester.injectElements(1); - assertFalse(tester.shouldFire(window)); - - tester.injectElements(2, 3, 4, 5); - assertTrue(tester.shouldFire(window)); - tester.fireIfShouldFire(window); - assertFalse(tester.shouldFire(window)); - } - - @Test - public void testRepeatedlyAfterFirstProcessingTime() throws Exception { - SimpleTriggerTester tester = - TriggerTester.forTrigger( - Repeatedly.forever( - AfterFirst.of( - AfterProcessingTime.pastFirstElementInPane() - .plusDelayOf(Duration.standardMinutes(15)), - AfterPane.elementCountAtLeast(5))), - new GlobalWindows()); - - GlobalWindow window = GlobalWindow.INSTANCE; - - tester.injectElements(1); - assertFalse(tester.shouldFire(window)); - - tester.advanceProcessingTime(new Instant(0).plus(Duration.standardMinutes(15))); - assertTrue(tester.shouldFire(window)); - tester.fireIfShouldFire(window); - assertFalse(tester.shouldFire(window)); - } - - @Test - public void testRepeatedlyElementCount() throws Exception { - SimpleTriggerTester tester = - TriggerTester.forTrigger( - Repeatedly.forever(AfterPane.elementCountAtLeast(5)), - new GlobalWindows()); - - GlobalWindow window = GlobalWindow.INSTANCE; - - tester.injectElements(1); - assertFalse(tester.shouldFire(window)); - - tester.injectElements(2, 3, 4, 5); - assertTrue(tester.shouldFire(window)); - tester.fireIfShouldFire(window); - assertFalse(tester.shouldFire(window)); - } - - @Test - public void testRepeatedlyProcessingTime() throws Exception { - SimpleTriggerTester tester = - TriggerTester.forTrigger( - Repeatedly.forever( - AfterProcessingTime.pastFirstElementInPane() - .plusDelayOf(Duration.standardMinutes(15))), - new GlobalWindows()); - - GlobalWindow window = GlobalWindow.INSTANCE; - - tester.injectElements(1); - assertFalse(tester.shouldFire(window)); - - tester.advanceProcessingTime(new Instant(0).plus(Duration.standardMinutes(15))); - assertTrue(tester.shouldFire(window)); - tester.fireIfShouldFire(window); - assertFalse(tester.shouldFire(window)); - } - - @Test public void testToString() { - Trigger trigger = Repeatedly.forever(new StubTrigger() { - @Override - public String toString() { - return "innerTrigger"; - } - }); + Trigger trigger = + Repeatedly.forever( + new StubTrigger() { + @Override + public String toString() { + return "innerTrigger"; + } + }); assertEquals("Repeatedly.forever(innerTrigger)", trigger.toString()); } - } diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/windowing/StubTrigger.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/windowing/StubTrigger.java index b258a791fd40..0fc74e7f2edc 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/windowing/StubTrigger.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/windowing/StubTrigger.java @@ -41,23 +41,6 @@ protected StubTrigger() { super(Lists.newArrayList()); } - @Override - protected void onOnlyFiring(TriggerContext context) throws Exception { - } - - @Override - public void onElement(OnElementContext c) throws Exception { - } - - @Override - public void onMerge(OnMergeContext c) throws Exception { - } - - @Override - public boolean shouldFire(TriggerContext context) throws Exception { - return false; - } - @Override protected Trigger getContinuationTrigger(List continuationTriggers) { return null; diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/windowing/TriggerTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/windowing/TriggerTest.java index cfc03b29beca..2602f7905db9 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/windowing/TriggerTest.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/windowing/TriggerTest.java @@ -58,12 +58,6 @@ private Trigger1(List subTriggers) { super(subTriggers); } - @Override - public void onElement(Trigger.OnElementContext c) { } - - @Override - public void onMerge(Trigger.OnMergeContext c) { } - @Override protected Trigger getContinuationTrigger( List continuationTriggers) { @@ -74,14 +68,6 @@ protected Trigger getContinuationTrigger( public Instant getWatermarkThatGuaranteesFiring(BoundedWindow window) { return null; } - - @Override - public boolean shouldFire(Trigger.TriggerContext context) throws Exception { - return false; - } - - @Override - public void onFire(Trigger.TriggerContext context) throws Exception { } } private static class Trigger2 extends Trigger { @@ -90,12 +76,6 @@ private Trigger2(List subTriggers) { super(subTriggers); } - @Override - public void onElement(Trigger.OnElementContext c) { } - - @Override - public void onMerge(Trigger.OnMergeContext c) { } - @Override protected Trigger getContinuationTrigger( List continuationTriggers) { @@ -106,13 +86,5 @@ protected Trigger getContinuationTrigger( public Instant getWatermarkThatGuaranteesFiring(BoundedWindow window) { return null; } - - @Override - public boolean shouldFire(Trigger.TriggerContext context) throws Exception { - return false; - } - - @Override - public void onFire(Trigger.TriggerContext context) throws Exception { } } } diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/util/ExecutableTriggerTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/util/ExecutableTriggerTest.java index 1e3a1ff8b9ee..befc07e54d9d 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/util/ExecutableTriggerTest.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/util/ExecutableTriggerTest.java @@ -91,16 +91,6 @@ protected StubTrigger(Trigger... subTriggers) { super(Arrays.asList(subTriggers)); } - @Override - public void onElement(OnElementContext c) throws Exception { } - - @Override - public void onMerge(OnMergeContext c) throws Exception { } - - @Override - public void clear(TriggerContext c) throws Exception { - } - @Override public Instant getWatermarkThatGuaranteesFiring(BoundedWindow window) { return BoundedWindow.TIMESTAMP_MAX_VALUE; @@ -115,13 +105,5 @@ public boolean isCompatible(Trigger other) { public Trigger getContinuationTrigger(List continuationTriggers) { return this; } - - @Override - public boolean shouldFire(TriggerContext c) { - return false; - } - - @Override - public void onFire(TriggerContext c) { } } } diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/util/FinishedTriggersBitSetTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/util/FinishedTriggersBitSetTest.java deleted file mode 100644 index 7f746206e581..000000000000 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/util/FinishedTriggersBitSetTest.java +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.beam.sdk.util; - -import static org.hamcrest.Matchers.not; -import static org.hamcrest.Matchers.theInstance; -import static org.junit.Assert.assertThat; - -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.JUnit4; - -/** - * Tests for {@link FinishedTriggersBitSet}. - */ -@RunWith(JUnit4.class) -public class FinishedTriggersBitSetTest { - /** - * Tests that after a trigger is set to finished, it reads back as finished. - */ - @Test - public void testSetGet() { - FinishedTriggersProperties.verifyGetAfterSet(FinishedTriggersBitSet.emptyWithCapacity(1)); - } - - /** - * Tests that clearing a trigger recursively clears all of that triggers subTriggers, but no - * others. - */ - @Test - public void testClearRecursively() { - FinishedTriggersProperties.verifyClearRecursively(FinishedTriggersBitSet.emptyWithCapacity(1)); - } - - @Test - public void testCopy() throws Exception { - FinishedTriggersBitSet finishedSet = FinishedTriggersBitSet.emptyWithCapacity(10); - assertThat(finishedSet.copy().getBitSet(), not(theInstance(finishedSet.getBitSet()))); - } -} diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/util/FinishedTriggersProperties.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/util/FinishedTriggersProperties.java deleted file mode 100644 index a66f74f87b94..000000000000 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/util/FinishedTriggersProperties.java +++ /dev/null @@ -1,110 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.beam.sdk.util; - -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; - -import org.apache.beam.sdk.transforms.windowing.AfterAll; -import org.apache.beam.sdk.transforms.windowing.AfterFirst; -import org.apache.beam.sdk.transforms.windowing.AfterPane; -import org.apache.beam.sdk.transforms.windowing.AfterProcessingTime; -import org.apache.beam.sdk.transforms.windowing.AfterWatermark; - -/** - * Generalized tests for {@link FinishedTriggers} implementations. - */ -public class FinishedTriggersProperties { - /** - * Tests that for the provided trigger and {@link FinishedTriggers}, when the trigger is set - * finished, it is correctly reported as finished. - */ - public static void verifyGetAfterSet(FinishedTriggers finishedSet, ExecutableTrigger trigger) { - assertFalse(finishedSet.isFinished(trigger)); - finishedSet.setFinished(trigger, true); - assertTrue(finishedSet.isFinished(trigger)); - } - - /** - * For a few arbitrary triggers, tests that when the trigger is set finished it is correctly - * reported as finished. - */ - public static void verifyGetAfterSet(FinishedTriggers finishedSet) { - ExecutableTrigger trigger = ExecutableTrigger.create(AfterAll.of( - AfterFirst.of(AfterPane.elementCountAtLeast(3), AfterWatermark.pastEndOfWindow()), - AfterAll.of( - AfterPane.elementCountAtLeast(10), AfterProcessingTime.pastFirstElementInPane()))); - - verifyGetAfterSet(finishedSet, trigger); - verifyGetAfterSet(finishedSet, trigger.subTriggers().get(0).subTriggers().get(1)); - verifyGetAfterSet(finishedSet, trigger.subTriggers().get(0)); - verifyGetAfterSet(finishedSet, trigger.subTriggers().get(1)); - verifyGetAfterSet(finishedSet, trigger.subTriggers().get(1).subTriggers().get(1)); - verifyGetAfterSet(finishedSet, trigger.subTriggers().get(1).subTriggers().get(0)); - } - - /** - * Tests that clearing a trigger recursively clears all of that triggers subTriggers, but no - * others. - */ - public static void verifyClearRecursively(FinishedTriggers finishedSet) { - ExecutableTrigger trigger = ExecutableTrigger.create(AfterAll.of( - AfterFirst.of(AfterPane.elementCountAtLeast(3), AfterWatermark.pastEndOfWindow()), - AfterAll.of( - AfterPane.elementCountAtLeast(10), AfterProcessingTime.pastFirstElementInPane()))); - - // Set them all finished. This method is not on a trigger as it makes no sense outside tests. - setFinishedRecursively(finishedSet, trigger); - assertTrue(finishedSet.isFinished(trigger)); - assertTrue(finishedSet.isFinished(trigger.subTriggers().get(0))); - assertTrue(finishedSet.isFinished(trigger.subTriggers().get(0).subTriggers().get(0))); - assertTrue(finishedSet.isFinished(trigger.subTriggers().get(0).subTriggers().get(1))); - - // Clear just the second AfterAll - finishedSet.clearRecursively(trigger.subTriggers().get(1)); - - // Check that the first and all that are still finished - assertTrue(finishedSet.isFinished(trigger)); - verifyFinishedRecursively(finishedSet, trigger.subTriggers().get(0)); - verifyUnfinishedRecursively(finishedSet, trigger.subTriggers().get(1)); - } - - private static void setFinishedRecursively( - FinishedTriggers finishedSet, ExecutableTrigger trigger) { - finishedSet.setFinished(trigger, true); - for (ExecutableTrigger subTrigger : trigger.subTriggers()) { - setFinishedRecursively(finishedSet, subTrigger); - } - } - - private static void verifyFinishedRecursively( - FinishedTriggers finishedSet, ExecutableTrigger trigger) { - assertTrue(finishedSet.isFinished(trigger)); - for (ExecutableTrigger subTrigger : trigger.subTriggers()) { - verifyFinishedRecursively(finishedSet, subTrigger); - } - } - - private static void verifyUnfinishedRecursively( - FinishedTriggers finishedSet, ExecutableTrigger trigger) { - assertFalse(finishedSet.isFinished(trigger)); - for (ExecutableTrigger subTrigger : trigger.subTriggers()) { - verifyUnfinishedRecursively(finishedSet, subTrigger); - } - } -} diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/util/FinishedTriggersSetTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/util/FinishedTriggersSetTest.java deleted file mode 100644 index 072d264f231c..000000000000 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/util/FinishedTriggersSetTest.java +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.beam.sdk.util; - -import static org.hamcrest.Matchers.not; -import static org.hamcrest.Matchers.theInstance; -import static org.junit.Assert.assertThat; - -import java.util.HashSet; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.JUnit4; - -/** - * Tests for {@link FinishedTriggersSet}. - */ -@RunWith(JUnit4.class) -public class FinishedTriggersSetTest { - /** - * Tests that after a trigger is set to finished, it reads back as finished. - */ - @Test - public void testSetGet() { - FinishedTriggersProperties.verifyGetAfterSet( - FinishedTriggersSet.fromSet(new HashSet())); - } - - /** - * Tests that clearing a trigger recursively clears all of that triggers subTriggers, but no - * others. - */ - @Test - public void testClearRecursively() { - FinishedTriggersProperties.verifyClearRecursively( - FinishedTriggersSet.fromSet(new HashSet())); - } - - @Test - public void testCopy() throws Exception { - FinishedTriggersSet finishedSet = - FinishedTriggersSet.fromSet(new HashSet()); - assertThat(finishedSet.copy().getFinishedTriggers(), - not(theInstance(finishedSet.getFinishedTriggers()))); - } -} diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/util/ReshuffleTriggerTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/util/ReshuffleTriggerTest.java index 83077f4cc37f..63c71ed0b72c 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/util/ReshuffleTriggerTest.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/util/ReshuffleTriggerTest.java @@ -18,15 +18,9 @@ package org.apache.beam.sdk.util; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; import org.apache.beam.sdk.transforms.windowing.BoundedWindow; -import org.apache.beam.sdk.transforms.windowing.FixedWindows; -import org.apache.beam.sdk.transforms.windowing.IntervalWindow; import org.apache.beam.sdk.transforms.windowing.Trigger; -import org.joda.time.Duration; -import org.joda.time.Instant; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; @@ -42,23 +36,6 @@ public static ReshuffleTrigger forTest() { return new ReshuffleTrigger<>(); } - @Test - public void testShouldFire() throws Exception { - TriggerTester tester = TriggerTester.forTrigger( - new ReshuffleTrigger(), FixedWindows.of(Duration.millis(100))); - IntervalWindow arbitraryWindow = new IntervalWindow(new Instant(300), new Instant(400)); - assertTrue(tester.shouldFire(arbitraryWindow)); - } - - @Test - public void testOnTimer() throws Exception { - TriggerTester tester = TriggerTester.forTrigger( - new ReshuffleTrigger(), FixedWindows.of(Duration.millis(100))); - IntervalWindow arbitraryWindow = new IntervalWindow(new Instant(100), new Instant(200)); - tester.fireIfShouldFire(arbitraryWindow); - assertFalse(tester.isMarkedFinished(arbitraryWindow)); - } - @Test public void testToString() { Trigger trigger = new ReshuffleTrigger<>(); diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/util/TriggerTester.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/util/TriggerTester.java deleted file mode 100644 index 5fe17addad7a..000000000000 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/util/TriggerTester.java +++ /dev/null @@ -1,410 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.beam.sdk.util; - -import static com.google.common.base.Preconditions.checkArgument; -import static com.google.common.base.Preconditions.checkNotNull; -import static org.junit.Assert.assertTrue; - -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import javax.annotation.Nullable; -import org.apache.beam.sdk.transforms.windowing.BoundedWindow; -import org.apache.beam.sdk.transforms.windowing.GlobalWindow; -import org.apache.beam.sdk.transforms.windowing.PaneInfo; -import org.apache.beam.sdk.transforms.windowing.Trigger; -import org.apache.beam.sdk.transforms.windowing.WindowFn; -import org.apache.beam.sdk.util.ActiveWindowSet.MergeCallback; -import org.apache.beam.sdk.util.TimerInternals.TimerData; -import org.apache.beam.sdk.util.WindowingStrategy.AccumulationMode; -import org.apache.beam.sdk.util.state.InMemoryTimerInternals; -import org.apache.beam.sdk.util.state.StateInternals; -import org.apache.beam.sdk.util.state.StateNamespace; -import org.apache.beam.sdk.util.state.StateNamespaces; -import org.apache.beam.sdk.util.state.StateNamespaces.WindowAndTriggerNamespace; -import org.apache.beam.sdk.util.state.StateNamespaces.WindowNamespace; -import org.apache.beam.sdk.util.state.TestInMemoryStateInternals; -import org.apache.beam.sdk.util.state.TimerCallback; -import org.apache.beam.sdk.values.TimestampedValue; -import org.joda.time.Duration; -import org.joda.time.Instant; - -/** - * Test utility that runs a {@link Trigger}, using in-memory stub implementation to provide - * the {@link StateInternals}. - * - * @param The type of windows being used. - */ -public class TriggerTester { - - /** - * A {@link TriggerTester} specialized to {@link Integer} values, so elements and timestamps - * can be conflated. Today, triggers should not observed the element type, so this is the - * only trigger tester that needs to be used. - */ - public static class SimpleTriggerTester - extends TriggerTester { - - private SimpleTriggerTester(WindowingStrategy windowingStrategy) throws Exception { - super(windowingStrategy); - } - - public void injectElements(int... values) throws Exception { - List> timestampedValues = - Lists.newArrayListWithCapacity(values.length); - for (int value : values) { - timestampedValues.add(TimestampedValue.of(value, new Instant(value))); - } - injectElements(timestampedValues); - } - - public SimpleTriggerTester withAllowedLateness(Duration allowedLateness) throws Exception { - return new SimpleTriggerTester<>( - windowingStrategy.withAllowedLateness(allowedLateness)); - } - } - - protected final WindowingStrategy windowingStrategy; - - private final TestInMemoryStateInternals stateInternals = - new TestInMemoryStateInternals(null /* key */); - private final InMemoryTimerInternals timerInternals = new InMemoryTimerInternals(); - private final TriggerContextFactory contextFactory; - private final WindowFn windowFn; - private final ActiveWindowSet activeWindows; - private final Map windowToMergeResult; - - /** - * An {@link ExecutableTrigger} built from the {@link Trigger} or {@link Trigger} - * under test. - */ - private final ExecutableTrigger executableTrigger; - - /** - * A map from a window and trigger to whether that trigger is finished for the window. - */ - private final Map finishedSets; - - public static SimpleTriggerTester forTrigger( - Trigger trigger, WindowFn windowFn) - throws Exception { - WindowingStrategy windowingStrategy = - WindowingStrategy.of(windowFn).withTrigger(trigger) - // Merging requires accumulation mode or early firings can break up a session. - // Not currently an issue with the tester (because we never GC) but we don't want - // mystery failures due to violating this need. - .withMode(windowFn.isNonMerging() - ? AccumulationMode.DISCARDING_FIRED_PANES - : AccumulationMode.ACCUMULATING_FIRED_PANES); - - return new SimpleTriggerTester<>(windowingStrategy); - } - - public static TriggerTester forAdvancedTrigger( - Trigger trigger, WindowFn windowFn) throws Exception { - WindowingStrategy strategy = - WindowingStrategy.of(windowFn).withTrigger(trigger) - // Merging requires accumulation mode or early firings can break up a session. - // Not currently an issue with the tester (because we never GC) but we don't want - // mystery failures due to violating this need. - .withMode(windowFn.isNonMerging() - ? AccumulationMode.DISCARDING_FIRED_PANES - : AccumulationMode.ACCUMULATING_FIRED_PANES); - - return new TriggerTester<>(strategy); - } - - protected TriggerTester(WindowingStrategy windowingStrategy) throws Exception { - this.windowingStrategy = windowingStrategy; - this.windowFn = windowingStrategy.getWindowFn(); - this.executableTrigger = windowingStrategy.getTrigger(); - this.finishedSets = new HashMap<>(); - - this.activeWindows = - windowFn.isNonMerging() - ? new NonMergingActiveWindowSet() - : new MergingActiveWindowSet(windowFn, stateInternals); - this.windowToMergeResult = new HashMap<>(); - - this.contextFactory = - new TriggerContextFactory<>(windowingStrategy.getWindowFn(), stateInternals, activeWindows); - } - - /** - * Instructs the trigger to clear its state for the given window. - */ - public void clearState(W window) throws Exception { - executableTrigger.invokeClear(contextFactory.base(window, - new TestTimers(windowNamespace(window)), executableTrigger, getFinishedSet(window))); - } - - /** - * Asserts that the trigger has actually cleared all of its state for the given window. Since - * the trigger under test is the root, this makes the assert for all triggers regardless - * of their position in the trigger tree. - */ - public void assertCleared(W window) { - for (StateNamespace untypedNamespace : stateInternals.getNamespacesInUse()) { - if (untypedNamespace instanceof WindowAndTriggerNamespace) { - @SuppressWarnings("unchecked") - WindowAndTriggerNamespace namespace = (WindowAndTriggerNamespace) untypedNamespace; - if (namespace.getWindow().equals(window)) { - Set tagsInUse = stateInternals.getTagsInUse(namespace); - assertTrue("Trigger has not cleared tags: " + tagsInUse, tagsInUse.isEmpty()); - } - } - } - } - - /** - * Returns {@code true} if the {@link Trigger} under test is finished for the given window. - */ - public boolean isMarkedFinished(W window) { - FinishedTriggers finishedSet = finishedSets.get(window); - if (finishedSet == null) { - return false; - } - - return finishedSet.isFinished(executableTrigger); - } - - private StateNamespace windowNamespace(W window) { - return StateNamespaces.window(windowFn.windowCoder(), checkNotNull(window)); - } - - /** - * Advance the input watermark to the specified time, then advance the output watermark as far as - * possible. - */ - public void advanceInputWatermark(Instant newInputWatermark) throws Exception { - // TODO: Should test timer firings: see https://issues.apache.org/jira/browse/BEAM-694 - timerInternals.advanceInputWatermark(TimerCallback.NO_OP, newInputWatermark); - } - - /** Advance the processing time to the specified time. */ - public void advanceProcessingTime(Instant newProcessingTime) throws Exception { - // TODO: Should test timer firings: see https://issues.apache.org/jira/browse/BEAM-694 - timerInternals.advanceProcessingTime(TimerCallback.NO_OP, newProcessingTime); - } - - /** - * Inject all the timestamped values (after passing through the window function) as if they - * arrived in a single chunk of a bundle (or work-unit). - */ - @SafeVarargs - public final void injectElements(TimestampedValue... values) throws Exception { - injectElements(Arrays.asList(values)); - } - - public final void injectElements(Collection> values) throws Exception { - for (TimestampedValue value : values) { - WindowTracing.trace("TriggerTester.injectElements: {}", value); - } - - List> windowedValues = Lists.newArrayListWithCapacity(values.size()); - - for (TimestampedValue input : values) { - try { - InputT value = input.getValue(); - Instant timestamp = input.getTimestamp(); - Collection assignedWindows = windowFn.assignWindows(new TestAssignContext( - windowFn, value, timestamp, GlobalWindow.INSTANCE)); - - for (W window : assignedWindows) { - activeWindows.addActiveForTesting(window); - - // Today, triggers assume onTimer firing at the watermark time, whether or not they - // explicitly set the timer themselves. So this tester must set it. - timerInternals.setTimer( - TimerData.of(windowNamespace(window), window.maxTimestamp(), TimeDomain.EVENT_TIME)); - } - - windowedValues.add(WindowedValue.of(value, timestamp, assignedWindows, PaneInfo.NO_FIRING)); - } catch (Exception e) { - throw new RuntimeException(e); - } - } - - for (WindowedValue windowedValue : windowedValues) { - for (BoundedWindow untypedWindow : windowedValue.getWindows()) { - // SDK is responsible for type safety - @SuppressWarnings("unchecked") - W window = mergeResult((W) untypedWindow); - - Trigger.OnElementContext context = contextFactory.createOnElementContext(window, - new TestTimers(windowNamespace(window)), windowedValue.getTimestamp(), - executableTrigger, getFinishedSet(window)); - - if (!context.trigger().isFinished()) { - executableTrigger.invokeOnElement(context); - } - } - } - } - - public boolean shouldFire(W window) throws Exception { - Trigger.TriggerContext context = contextFactory.base( - window, - new TestTimers(windowNamespace(window)), - executableTrigger, getFinishedSet(window)); - executableTrigger.getSpec().prefetchShouldFire(context.state()); - return executableTrigger.invokeShouldFire(context); - } - - public void fireIfShouldFire(W window) throws Exception { - Trigger.TriggerContext context = contextFactory.base( - window, - new TestTimers(windowNamespace(window)), - executableTrigger, getFinishedSet(window)); - - executableTrigger.getSpec().prefetchShouldFire(context.state()); - if (executableTrigger.invokeShouldFire(context)) { - executableTrigger.getSpec().prefetchOnFire(context.state()); - executableTrigger.invokeOnFire(context); - if (context.trigger().isFinished()) { - activeWindows.remove(window); - executableTrigger.invokeClear(context); - } - } - } - - public void setSubTriggerFinishedForWindow(int subTriggerIndex, W window, boolean value) { - getFinishedSet(window).setFinished(executableTrigger.subTriggers().get(subTriggerIndex), value); - } - - /** - * Invokes merge from the {@link WindowFn} a single time and passes the resulting merge - * events on to the trigger under test. Does not persist the fact that merging happened, - * since it is just to test the trigger's {@code OnMerge} method. - */ - public final void mergeWindows() throws Exception { - windowToMergeResult.clear(); - activeWindows.merge(new MergeCallback() { - @Override - public void prefetchOnMerge(Collection toBeMerged, W mergeResult) throws Exception {} - - @Override - public void onMerge(Collection toBeMerged, W mergeResult) throws Exception { - List activeToBeMerged = new ArrayList(); - for (W window : toBeMerged) { - windowToMergeResult.put(window, mergeResult); - if (activeWindows.isActive(window)) { - activeToBeMerged.add(window); - } - } - Map mergingFinishedSets = - Maps.newHashMapWithExpectedSize(activeToBeMerged.size()); - for (W oldWindow : activeToBeMerged) { - mergingFinishedSets.put(oldWindow, getFinishedSet(oldWindow)); - } - executableTrigger.invokeOnMerge(contextFactory.createOnMergeContext(mergeResult, - new TestTimers(windowNamespace(mergeResult)), executableTrigger, - getFinishedSet(mergeResult), mergingFinishedSets)); - timerInternals.setTimer(TimerData.of( - windowNamespace(mergeResult), mergeResult.maxTimestamp(), TimeDomain.EVENT_TIME)); - } - }); - } - - public W mergeResult(W window) { - W result = windowToMergeResult.get(window); - return result == null ? window : result; - } - - private FinishedTriggers getFinishedSet(W window) { - FinishedTriggers finishedSet = finishedSets.get(window); - if (finishedSet == null) { - finishedSet = FinishedTriggersSet.fromSet(new HashSet()); - finishedSets.put(window, finishedSet); - } - return finishedSet; - } - - private static class TestAssignContext - extends WindowFn.AssignContext { - private Object element; - private Instant timestamp; - private BoundedWindow window; - - public TestAssignContext( - WindowFn windowFn, Object element, Instant timestamp, BoundedWindow window) { - windowFn.super(); - this.element = element; - this.timestamp = timestamp; - this.window = window; - } - - @Override - public Object element() { - return element; - } - - @Override - public Instant timestamp() { - return timestamp; - } - - @Override - public BoundedWindow window() { - return window; - } - } - - private class TestTimers implements Timers { - private final StateNamespace namespace; - - public TestTimers(StateNamespace namespace) { - checkArgument(namespace instanceof WindowNamespace); - this.namespace = namespace; - } - - @Override - public void setTimer(Instant timestamp, TimeDomain timeDomain) { - timerInternals.setTimer(TimerData.of(namespace, timestamp, timeDomain)); - } - - @Override - public void deleteTimer(Instant timestamp, TimeDomain timeDomain) { - timerInternals.deleteTimer(TimerData.of(namespace, timestamp, timeDomain)); - } - - @Override - public Instant currentProcessingTime() { - return timerInternals.currentProcessingTime(); - } - - @Override - @Nullable - public Instant currentSynchronizedProcessingTime() { - return timerInternals.currentSynchronizedProcessingTime(); - } - - @Override - public Instant currentEventTime() { - return timerInternals.currentInputWatermarkTime(); - } - } -} From 90c30cba9d5beaf19de5f90419df353a0511f8b5 Mon Sep 17 00:00:00 2001 From: Kenneth Knowles Date: Mon, 24 Oct 2016 15:36:45 -0700 Subject: [PATCH 2/2] Update DataflowRunner worker image --- .../java/org/apache/beam/runners/dataflow/DataflowRunner.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowRunner.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowRunner.java index 89c364ea0a84..2324196ab379 100644 --- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowRunner.java +++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowRunner.java @@ -208,9 +208,9 @@ public class DataflowRunner extends PipelineRunner { // Default Docker container images that execute Dataflow worker harness, residing in Google // Container Registry, separately for Batch and Streaming. public static final String BATCH_WORKER_HARNESS_CONTAINER_IMAGE = - "dataflow.gcr.io/v1beta3/beam-java-batch:beam-master-20161017"; + "dataflow.gcr.io/v1beta3/beam-java-batch:beam-master-20161024"; public static final String STREAMING_WORKER_HARNESS_CONTAINER_IMAGE = - "dataflow.gcr.io/v1beta3/beam-java-streaming:beam-master-20161017"; + "dataflow.gcr.io/v1beta3/beam-java-streaming:beam-master-20161024"; // The limit of CreateJob request size. private static final int CREATE_JOB_REQUEST_LIMIT_BYTES = 10 * 1024 * 1024;