From ed012d3d73e261b8854fb9baecaca7a75494f1d3 Mon Sep 17 00:00:00 2001 From: Ventura Del Monte Date: Wed, 22 Feb 2017 14:55:17 +0100 Subject: [PATCH] [FLINK-5157] [streaming] Introduce ProcessAllWindowFunction --- .../api/datastream/AllWindowedStream.java | 394 +++++++++++++++- .../FoldApplyProcessAllWindowFunction.java | 120 +++++ .../windowing/ProcessAllWindowFunction.java | 59 +++ .../ReduceApplyProcessAllWindowFunction.java | 80 ++++ .../RichProcessAllWindowFunction.java | 84 ++++ ...rnalAggregateProcessAllWindowFunction.java | 83 ++++ ...ernalIterableProcessAllWindowFunction.java | 63 +++ ...alSingleValueProcessAllWindowFunction.java | 65 +++ ...ernalSingleValueProcessWindowFunction.java | 3 +- .../FoldApplyProcessWindowFunctionTest.java | 99 ++++ .../operators/StateDescriptorPassingTest.java | 19 + .../functions/InternalWindowFunctionTest.java | 193 +++++++- .../windowing/AllWindowTranslationTest.java | 445 ++++++++++++++++++ .../api/scala/AllWindowedStream.scala | 181 ++++++- .../function/ProcessAllWindowFunction.scala | 59 +++ .../RichProcessAllWindowFunction.scala | 86 ++++ .../ScalaProcessWindowFunctionWrapper.scala | 85 +++- .../api/scala/AllWindowTranslationTest.scala | 410 +++++++++++++++- .../api/scala/WindowFoldITCase.scala | 60 ++- .../api/scala/WindowFunctionITCase.scala | 51 +- .../api/scala/WindowReduceITCase.scala | 59 ++- ...IdentityRichProcessAllWindowFunction.scala | 81 ++++ .../streaming/runtime/WindowFoldITCase.java | 73 +++ 23 files changed, 2821 insertions(+), 31 deletions(-) create mode 100644 flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/FoldApplyProcessAllWindowFunction.java create mode 100644 flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/ProcessAllWindowFunction.java create mode 100644 flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/ReduceApplyProcessAllWindowFunction.java create mode 100644 flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/RichProcessAllWindowFunction.java create mode 100644 flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/functions/InternalAggregateProcessAllWindowFunction.java create mode 100644 flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/functions/InternalIterableProcessAllWindowFunction.java create mode 100644 flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/functions/InternalSingleValueProcessAllWindowFunction.java create mode 100644 flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/function/ProcessAllWindowFunction.scala create mode 100644 flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/function/RichProcessAllWindowFunction.scala create mode 100644 flink-streaming-scala/src/test/scala/org/apache/flink/streaming/api/scala/testutils/CheckingIdentityRichProcessAllWindowFunction.scala diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/AllWindowedStream.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/AllWindowedStream.java index c3c7424aa7823f..41b6d6fff0fa1c 100644 --- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/AllWindowedStream.java +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/AllWindowedStream.java @@ -39,9 +39,12 @@ import org.apache.flink.streaming.api.functions.aggregation.SumAggregator; import org.apache.flink.streaming.api.functions.windowing.AggregateApplyAllWindowFunction; import org.apache.flink.streaming.api.functions.windowing.AllWindowFunction; +import org.apache.flink.streaming.api.functions.windowing.FoldApplyProcessAllWindowFunction; import org.apache.flink.streaming.api.functions.windowing.PassThroughAllWindowFunction; import org.apache.flink.streaming.api.functions.windowing.FoldApplyAllWindowFunction; +import org.apache.flink.streaming.api.functions.windowing.ProcessAllWindowFunction; import org.apache.flink.streaming.api.functions.windowing.ReduceApplyAllWindowFunction; +import org.apache.flink.streaming.api.functions.windowing.ReduceApplyProcessAllWindowFunction; import org.apache.flink.streaming.api.operators.OneInputStreamOperator; import org.apache.flink.streaming.api.windowing.assigners.MergingWindowAssigner; import org.apache.flink.streaming.api.windowing.assigners.WindowAssigner; @@ -51,8 +54,12 @@ import org.apache.flink.streaming.api.windowing.windows.Window; import org.apache.flink.streaming.runtime.operators.windowing.EvictingWindowOperator; import org.apache.flink.streaming.runtime.operators.windowing.WindowOperator; +import org.apache.flink.streaming.runtime.operators.windowing.functions.InternalAggregateProcessAllWindowFunction; import org.apache.flink.streaming.runtime.operators.windowing.functions.InternalIterableAllWindowFunction; +import org.apache.flink.streaming.runtime.operators.windowing.functions.InternalIterableProcessAllWindowFunction; import org.apache.flink.streaming.runtime.operators.windowing.functions.InternalSingleValueAllWindowFunction; +import org.apache.flink.streaming.runtime.operators.windowing.functions.InternalSingleValueProcessAllWindowFunction; +import org.apache.flink.streaming.runtime.operators.windowing.functions.InternalWindowFunction; import org.apache.flink.streaming.runtime.streamrecord.StreamElementSerializer; import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; @@ -286,6 +293,102 @@ public SingleOutputStreamOperator reduce(ReduceFunction reduceFunction return input.transform(opName, resultType, operator).forceNonParallel(); } + /** + * Applies the given window function to each window. The window function is called for each + * evaluation of the window for each key individually. The output of the window function is + * interpreted as a regular non-windowed stream. + * + *

+ * Arriving data is incrementally aggregated using the given reducer. + * + * @param reduceFunction The reduce function that is used for incremental aggregation. + * @param function The process window function. + * @return The data stream that is the result of applying the window function to the window. + */ + @PublicEvolving + public SingleOutputStreamOperator reduce( + ReduceFunction reduceFunction, + ProcessAllWindowFunction function) { + + TypeInformation resultType = TypeExtractor.getUnaryOperatorReturnType( + function, ProcessAllWindowFunction.class, true, true, input.getType(), null, false); + + return reduce(reduceFunction, function, resultType); + } + + /** + * Applies the given window function to each window. The window function is called for each + * evaluation of the window for each key individually. The output of the window function is + * interpreted as a regular non-windowed stream. + * + *

+ * Arriving data is incrementally aggregated using the given reducer. + * + * @param reduceFunction The reduce function that is used for incremental aggregation. + * @param function The process window function. + * @param resultType Type information for the result type of the window function + * @return The data stream that is the result of applying the window function to the window. + */ + @PublicEvolving + public SingleOutputStreamOperator reduce(ReduceFunction reduceFunction, ProcessAllWindowFunction function, TypeInformation resultType) { + if (reduceFunction instanceof RichFunction) { + throw new UnsupportedOperationException("ReduceFunction of reduce can not be a RichFunction."); + } + + //clean the closures + function = input.getExecutionEnvironment().clean(function); + reduceFunction = input.getExecutionEnvironment().clean(reduceFunction); + + String callLocation = Utils.getCallLocationName(); + String udfName = "AllWindowedStream." + callLocation; + + String opName; + KeySelector keySel = input.getKeySelector(); + + OneInputStreamOperator operator; + + if (evictor != null) { + @SuppressWarnings({"unchecked", "rawtypes"}) + TypeSerializer> streamRecordSerializer = + (TypeSerializer>) new StreamElementSerializer(input.getType().createSerializer(getExecutionEnvironment().getConfig())); + + ListStateDescriptor> stateDesc = + new ListStateDescriptor<>("window-contents", streamRecordSerializer); + + opName = "TriggerWindow(" + windowAssigner + ", " + stateDesc + ", " + trigger + ", " + evictor + ", " + udfName + ")"; + + operator = + new EvictingWindowOperator<>(windowAssigner, + windowAssigner.getWindowSerializer(getExecutionEnvironment().getConfig()), + keySel, + input.getKeyType().createSerializer(getExecutionEnvironment().getConfig()), + stateDesc, + new InternalIterableProcessAllWindowFunction<>(new ReduceApplyProcessAllWindowFunction<>(reduceFunction, function)), + trigger, + evictor, + allowedLateness); + + } else { + ReducingStateDescriptor stateDesc = new ReducingStateDescriptor<>("window-contents", + reduceFunction, + input.getType().createSerializer(getExecutionEnvironment().getConfig())); + + opName = "TriggerWindow(" + windowAssigner + ", " + stateDesc + ", " + trigger + ", " + udfName + ")"; + + operator = + new WindowOperator<>(windowAssigner, + windowAssigner.getWindowSerializer(getExecutionEnvironment().getConfig()), + keySel, + input.getKeyType().createSerializer(getExecutionEnvironment().getConfig()), + stateDesc, + new InternalSingleValueProcessAllWindowFunction<>(function), + trigger, + allowedLateness); + } + + return input.transform(opName, resultType, operator).forceNonParallel(); + } + // ------------------------------------------------------------------------ // AggregateFunction // ------------------------------------------------------------------------ @@ -478,6 +581,135 @@ public SingleOutputStreamOperator aggregate( return input.transform(opName, resultType, operator).forceNonParallel(); } + /** + * Applies the given window function to each window. The window function is called for each + * evaluation of the window for each key individually. The output of the window function is + * interpreted as a regular non-windowed stream. + * + *

Arriving data is incrementally aggregated using the given aggregate function. This means + * that the window function typically has only a single value to process when called. + * + * @param aggFunction The aggregate function that is used for incremental aggregation. + * @param windowFunction The process window function. + * + * @return The data stream that is the result of applying the window function to the window. + * + * @param The type of the AggregateFunction's accumulator + * @param The type of AggregateFunction's result, and the WindowFunction's input + * @param The type of the elements in the resulting stream, equal to the + * WindowFunction's result type + */ + public SingleOutputStreamOperator aggregate( + AggregateFunction aggFunction, + ProcessAllWindowFunction windowFunction) { + + checkNotNull(aggFunction, "aggFunction"); + checkNotNull(windowFunction, "windowFunction"); + + TypeInformation accumulatorType = TypeExtractor.getAggregateFunctionAccumulatorType( + aggFunction, input.getType(), null, false); + + TypeInformation aggResultType = TypeExtractor.getAggregateFunctionReturnType( + aggFunction, input.getType(), null, false); + + TypeInformation resultType = TypeExtractor.getUnaryOperatorReturnType( + windowFunction, ProcessAllWindowFunction.class, true, true, aggResultType, null, false); + + return aggregate(aggFunction, windowFunction, accumulatorType, aggResultType, resultType); + } + + /** + * Applies the given window function to each window. The window function is called for each + * evaluation of the window for each key individually. The output of the window function is + * interpreted as a regular non-windowed stream. + * + *

Arriving data is incrementally aggregated using the given aggregate function. This means + * that the window function typically has only a single value to process when called. + * + * @param aggregateFunction The aggregation function that is used for incremental aggregation. + * @param windowFunction The process window function. + * @param accumulatorType Type information for the internal accumulator type of the aggregation function + * @param resultType Type information for the result type of the window function + * + * @return The data stream that is the result of applying the window function to the window. + * + * @param The type of the AggregateFunction's accumulator + * @param The type of AggregateFunction's result, and the WindowFunction's input + * @param The type of the elements in the resulting stream, equal to the + * WindowFunction's result type + */ + public SingleOutputStreamOperator aggregate( + AggregateFunction aggregateFunction, + ProcessAllWindowFunction windowFunction, + TypeInformation accumulatorType, + TypeInformation aggregateResultType, + TypeInformation resultType) { + + checkNotNull(aggregateFunction, "aggregateFunction"); + checkNotNull(windowFunction, "windowFunction"); + checkNotNull(accumulatorType, "accumulatorType"); + checkNotNull(aggregateResultType, "aggregateResultType"); + checkNotNull(resultType, "resultType"); + + if (aggregateFunction instanceof RichFunction) { + throw new UnsupportedOperationException("This aggregate function cannot be a RichFunction."); + } + + //clean the closures + windowFunction = input.getExecutionEnvironment().clean(windowFunction); + aggregateFunction = input.getExecutionEnvironment().clean(aggregateFunction); + + final String callLocation = Utils.getCallLocationName(); + final String udfName = "AllWindowedStream." + callLocation; + + final String opName; + final KeySelector keySel = input.getKeySelector(); + + OneInputStreamOperator operator; + + if (evictor != null) { + @SuppressWarnings({"unchecked", "rawtypes"}) + TypeSerializer> streamRecordSerializer = + (TypeSerializer>) new StreamElementSerializer( + input.getType().createSerializer(getExecutionEnvironment().getConfig())); + + ListStateDescriptor> stateDesc = + new ListStateDescriptor<>("window-contents", streamRecordSerializer); + + opName = "TriggerWindow(" + windowAssigner + ", " + stateDesc + ", " + trigger + ", " + evictor + ", " + udfName + ")"; + + operator = new EvictingWindowOperator<>(windowAssigner, + windowAssigner.getWindowSerializer(getExecutionEnvironment().getConfig()), + keySel, + input.getKeyType().createSerializer(getExecutionEnvironment().getConfig()), + stateDesc, + new InternalAggregateProcessAllWindowFunction<>(aggregateFunction, windowFunction), + trigger, + evictor, + allowedLateness); + + } else { + AggregatingStateDescriptor stateDesc = new AggregatingStateDescriptor<>( + "window-contents", + aggregateFunction, + accumulatorType.createSerializer(getExecutionEnvironment().getConfig())); + + opName = "TriggerWindow(" + windowAssigner + ", " + stateDesc + ", " + trigger + ", " + udfName + ")"; + + operator = new WindowOperator<>( + windowAssigner, + windowAssigner.getWindowSerializer(getExecutionEnvironment().getConfig()), + keySel, + input.getKeyType().createSerializer(getExecutionEnvironment().getConfig()), + stateDesc, + new InternalSingleValueProcessAllWindowFunction<>(windowFunction), + trigger, + allowedLateness); + } + + return input.transform(opName, resultType, operator).forceNonParallel(); + } + // ------------------------------------------------------------------------ // FoldFunction // ------------------------------------------------------------------------ @@ -625,13 +857,119 @@ public SingleOutputStreamOperator fold(ACC initialValue, return input.transform(opName, resultType, operator).forceNonParallel(); } + /** + * Applies the given window function to each window. The window function is called for each + * evaluation of the window for each key individually. The output of the window function is + * interpreted as a regular non-windowed stream. + * + *

+ * Arriving data is incrementally aggregated using the given fold function. + * + * @param initialValue The initial value of the fold. + * @param foldFunction The fold function that is used for incremental aggregation. + * @param function The window function. + * @return The data stream that is the result of applying the window function to the window. + */ + @PublicEvolving + public SingleOutputStreamOperator fold(ACC initialValue, FoldFunction foldFunction, ProcessAllWindowFunction function) { + + TypeInformation foldAccumulatorType = TypeExtractor.getFoldReturnTypes(foldFunction, input.getType(), + Utils.getCallLocationName(), true); + + TypeInformation resultType = TypeExtractor.getUnaryOperatorReturnType( + function, ProcessAllWindowFunction.class, true, true, foldAccumulatorType, null, false); + + return fold(initialValue, foldFunction, function, foldAccumulatorType, resultType); + } + + /** + * Applies the given window function to each window. The window function is called for each + * evaluation of the window for each key individually. The output of the window function is + * interpreted as a regular non-windowed stream. + * + *

+ * Arriving data is incrementally aggregated using the given fold function. + * + * @param initialValue The initial value of the fold. + * @param foldFunction The fold function that is used for incremental aggregation. + * @param function The process window function. + * @param foldAccumulatorType Type information for the result type of the fold function + * @param resultType Type information for the result type of the window function + * @return The data stream that is the result of applying the window function to the window. + */ + @PublicEvolving + public SingleOutputStreamOperator fold(ACC initialValue, + FoldFunction foldFunction, + ProcessAllWindowFunction function, + TypeInformation foldAccumulatorType, + TypeInformation resultType) { + if (foldFunction instanceof RichFunction) { + throw new UnsupportedOperationException("FoldFunction of fold can not be a RichFunction."); + } + if (windowAssigner instanceof MergingWindowAssigner) { + throw new UnsupportedOperationException("Fold cannot be used with a merging WindowAssigner."); + } + + //clean the closures + function = input.getExecutionEnvironment().clean(function); + foldFunction = input.getExecutionEnvironment().clean(foldFunction); + + String callLocation = Utils.getCallLocationName(); + String udfName = "AllWindowedStream." + callLocation; + + String opName; + KeySelector keySel = input.getKeySelector(); + + OneInputStreamOperator operator; + + if (evictor != null) { + @SuppressWarnings({"unchecked", "rawtypes"}) + TypeSerializer> streamRecordSerializer = + (TypeSerializer>) new StreamElementSerializer(input.getType().createSerializer(getExecutionEnvironment().getConfig())); + + ListStateDescriptor> stateDesc = + new ListStateDescriptor<>("window-contents", streamRecordSerializer); + + opName = "TriggerWindow(" + windowAssigner + ", " + stateDesc + ", " + trigger + ", " + evictor + ", " + udfName + ")"; + + operator = + new EvictingWindowOperator<>(windowAssigner, + windowAssigner.getWindowSerializer(getExecutionEnvironment().getConfig()), + keySel, + input.getKeyType().createSerializer(getExecutionEnvironment().getConfig()), + stateDesc, + new InternalIterableProcessAllWindowFunction<>(new FoldApplyProcessAllWindowFunction<>(initialValue, foldFunction, function, foldAccumulatorType)), + trigger, + evictor, + allowedLateness); + + } else { + FoldingStateDescriptor stateDesc = new FoldingStateDescriptor<>("window-contents", + initialValue, foldFunction, foldAccumulatorType.createSerializer(getExecutionEnvironment().getConfig())); + + opName = "TriggerWindow(" + windowAssigner + ", " + stateDesc + ", " + trigger + ", " + udfName + ")"; + + operator = + new WindowOperator<>(windowAssigner, + windowAssigner.getWindowSerializer(getExecutionEnvironment().getConfig()), + keySel, + input.getKeyType().createSerializer(getExecutionEnvironment().getConfig()), + stateDesc, + new InternalSingleValueProcessAllWindowFunction<>(function), + trigger, + allowedLateness); + } + + return input.transform(opName, resultType, operator).forceNonParallel(); + } + // ------------------------------------------------------------------------ // Apply (Window Function) // ------------------------------------------------------------------------ /** * Applies the given window function to each window. The window function is called for each - * evaluation of the window for each key individually. The output of the window function is + * evaluation of the window. The output of the window function is * interpreted as a regular non-windowed stream. * *

@@ -642,15 +980,16 @@ public SingleOutputStreamOperator fold(ACC initialValue, * @return The data stream that is the result of applying the window function to the window. */ public SingleOutputStreamOperator apply(AllWindowFunction function) { + String callLocation = Utils.getCallLocationName(); + function = input.getExecutionEnvironment().clean(function); TypeInformation resultType = TypeExtractor.getUnaryOperatorReturnType( function, AllWindowFunction.class, true, true, getInputType(), null, false); - - return apply(function, resultType); + return apply(new InternalIterableAllWindowFunction<>(function), resultType, callLocation); } /** * Applies the given window function to each window. The window function is called for each - * evaluation of the window for each key individually. The output of the window function is + * evaluation of the window. The output of the window function is * interpreted as a regular non-windowed stream. * *

@@ -658,15 +997,54 @@ public SingleOutputStreamOperator apply(AllWindowFunction functi * is evaluated, as the function provides no means of incremental aggregation. * * @param function The window function. - * @param resultType Type information for the result type of the window function * @return The data stream that is the result of applying the window function to the window. */ public SingleOutputStreamOperator apply(AllWindowFunction function, TypeInformation resultType) { + String callLocation = Utils.getCallLocationName(); + function = input.getExecutionEnvironment().clean(function); + return apply(new InternalIterableAllWindowFunction<>(function), resultType, callLocation); + } - //clean the closure + /** + * Applies the given window function to each window. The window function is called for each + * evaluation of the window. The output of the window function is + * interpreted as a regular non-windowed stream. + * + *

+ * Not that this function requires that all data in the windows is buffered until the window + * is evaluated, as the function provides no means of incremental aggregation. + * + * @param function The process window function. + * @return The data stream that is the result of applying the window function to the window. + */ + public SingleOutputStreamOperator process(ProcessAllWindowFunction function) { + String callLocation = Utils.getCallLocationName(); function = input.getExecutionEnvironment().clean(function); + TypeInformation resultType = TypeExtractor.getUnaryOperatorReturnType( + function, ProcessAllWindowFunction.class, true, true, getInputType(), null, false); + return apply(new InternalIterableProcessAllWindowFunction<>(function), resultType, callLocation); + } + /** + * Applies the given window function to each window. The window function is called for each + * evaluation of the window. The output of the window function is + * interpreted as a regular non-windowed stream. + * + *

+ * Not that this function requires that all data in the windows is buffered until the window + * is evaluated, as the function provides no means of incremental aggregation. + * + * @param function The process window function. + * @return The data stream that is the result of applying the window function to the window. + */ + public SingleOutputStreamOperator process(ProcessAllWindowFunction function, TypeInformation resultType) { String callLocation = Utils.getCallLocationName(); + function = input.getExecutionEnvironment().clean(function); + return apply(new InternalIterableProcessAllWindowFunction<>(function), resultType, callLocation); + } + + private SingleOutputStreamOperator apply(InternalWindowFunction, R, Byte, W> function, TypeInformation resultType, String callLocation) { + String udfName = "AllWindowedStream." + callLocation; String opName; @@ -690,7 +1068,7 @@ public SingleOutputStreamOperator apply(AllWindowFunction functi keySel, input.getKeyType().createSerializer(getExecutionEnvironment().getConfig()), stateDesc, - new InternalIterableAllWindowFunction<>(function), + function, trigger, evictor, allowedLateness); @@ -707,7 +1085,7 @@ public SingleOutputStreamOperator apply(AllWindowFunction functi keySel, input.getKeyType().createSerializer(getExecutionEnvironment().getConfig()), stateDesc, - new InternalIterableAllWindowFunction<>(function), + function, trigger, allowedLateness); } diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/FoldApplyProcessAllWindowFunction.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/FoldApplyProcessAllWindowFunction.java new file mode 100644 index 00000000000000..5ac676661cfde5 --- /dev/null +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/FoldApplyProcessAllWindowFunction.java @@ -0,0 +1,120 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.flink.streaming.api.functions.windowing; + +import org.apache.flink.annotation.Internal; +import org.apache.flink.api.common.ExecutionConfig; +import org.apache.flink.api.common.functions.FoldFunction; +import org.apache.flink.api.common.functions.RuntimeContext; +import org.apache.flink.api.common.functions.util.FunctionUtils; +import org.apache.flink.api.common.typeinfo.TypeInformation; +import org.apache.flink.api.common.typeutils.TypeSerializer; +import org.apache.flink.configuration.Configuration; +import org.apache.flink.core.memory.DataInputViewStreamWrapper; +import org.apache.flink.core.memory.DataOutputViewStreamWrapper; +import org.apache.flink.streaming.api.operators.OutputTypeConfigurable; +import org.apache.flink.streaming.api.windowing.windows.Window; +import org.apache.flink.util.Collector; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.util.Collections; + +@Internal +public class FoldApplyProcessAllWindowFunction + extends RichProcessAllWindowFunction + implements OutputTypeConfigurable { + + private static final long serialVersionUID = 1L; + + private final FoldFunction foldFunction; + private final ProcessAllWindowFunction windowFunction; + + private byte[] serializedInitialValue; + private TypeSerializer accSerializer; + private final TypeInformation accTypeInformation; + private transient ACC initialValue; + + public FoldApplyProcessAllWindowFunction(ACC initialValue, FoldFunction foldFunction, ProcessAllWindowFunction windowFunction, TypeInformation accTypeInformation) { + this.windowFunction = windowFunction; + this.foldFunction = foldFunction; + this.initialValue = initialValue; + this.accTypeInformation = accTypeInformation; + } + + @Override + public void open(Configuration configuration) throws Exception { + FunctionUtils.openFunction(this.windowFunction, configuration); + + if (serializedInitialValue == null) { + throw new RuntimeException("No initial value was serialized for the fold " + + "window function. Probably the setOutputType method was not called."); + } + + ByteArrayInputStream bais = new ByteArrayInputStream(serializedInitialValue); + DataInputViewStreamWrapper in = new DataInputViewStreamWrapper(bais); + initialValue = accSerializer.deserialize(in); + } + + @Override + public void close() throws Exception { + FunctionUtils.closeFunction(this.windowFunction); + } + + @Override + public void setRuntimeContext(RuntimeContext t) { + super.setRuntimeContext(t); + + FunctionUtils.setFunctionRuntimeContext(this.windowFunction, t); + } + + @Override + public void process(final Context context, Iterable values, Collector out) throws Exception { + ACC result = accSerializer.copy(initialValue); + + for (T val : values) { + result = foldFunction.fold(result, val); + } + + windowFunction.process(windowFunction.new Context() { + @Override + public W window() { + return context.window(); + } + }, Collections.singletonList(result), out); + } + + @Override + public void setOutputType(TypeInformation outTypeInfo, ExecutionConfig executionConfig) { + accSerializer = accTypeInformation.createSerializer(executionConfig); + + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + DataOutputViewStreamWrapper out = new DataOutputViewStreamWrapper(baos); + + try { + accSerializer.serialize(initialValue, out); + } catch (IOException ioe) { + throw new RuntimeException("Unable to serialize initial value of type " + + initialValue.getClass().getSimpleName() + " of fold window function.", ioe); + } + + serializedInitialValue = baos.toByteArray(); + } + +} diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/ProcessAllWindowFunction.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/ProcessAllWindowFunction.java new file mode 100644 index 00000000000000..622e020624ab91 --- /dev/null +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/ProcessAllWindowFunction.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.streaming.api.functions.windowing; + +import org.apache.flink.annotation.PublicEvolving; +import org.apache.flink.api.common.functions.Function; +import org.apache.flink.streaming.api.windowing.windows.Window; +import org.apache.flink.util.Collector; + +/** + * Base abstract class for functions that are evaluated over non-keyed windows using a context + * for retrieving extra information. + * + * @param The type of the input value. + * @param The type of the output value. + * @param The type of {@code Window} that this window function can be applied on. + */ +@PublicEvolving +public abstract class ProcessAllWindowFunction implements Function { + + private static final long serialVersionUID = 1L; + + /** + * Evaluates the window and outputs none or several elements. + * + * @param context The context in which the window is being evaluated. + * @param elements The elements in the window being evaluated. + * @param out A collector for emitting elements. + * + * @throws Exception The function may throw exceptions to fail the program and trigger recovery. + */ + public abstract void process(Context context, Iterable elements, Collector out) throws Exception; + + /** + * The context holding window metadata + */ + public abstract class Context { + /** + * @return The window that is being evaluated. + */ + public abstract W window(); + } +} diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/ReduceApplyProcessAllWindowFunction.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/ReduceApplyProcessAllWindowFunction.java new file mode 100644 index 00000000000000..142c71e58ce42b --- /dev/null +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/ReduceApplyProcessAllWindowFunction.java @@ -0,0 +1,80 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.flink.streaming.api.functions.windowing; + +import org.apache.flink.annotation.Internal; +import org.apache.flink.api.common.functions.ReduceFunction; +import org.apache.flink.api.common.functions.RuntimeContext; +import org.apache.flink.api.common.functions.util.FunctionUtils; +import org.apache.flink.configuration.Configuration; +import org.apache.flink.streaming.api.windowing.windows.Window; +import org.apache.flink.util.Collector; + +import java.util.Collections; + +@Internal +public class ReduceApplyProcessAllWindowFunction + extends RichProcessAllWindowFunction { + + private static final long serialVersionUID = 1L; + + private final ReduceFunction reduceFunction; + private final ProcessAllWindowFunction windowFunction; + + public ReduceApplyProcessAllWindowFunction(ReduceFunction reduceFunction, ProcessAllWindowFunction windowFunction) { + this.windowFunction = windowFunction; + this.reduceFunction = reduceFunction; + } + + @Override + public void process(final Context context, Iterable input, Collector out) throws Exception { + + T curr = null; + for (T val: input) { + if (curr == null) { + curr = val; + } else { + curr = reduceFunction.reduce(curr, val); + } + } + windowFunction.process(windowFunction.new Context() { + @Override + public W window() { + return context.window(); + } + }, Collections.singletonList(curr), out); + } + + @Override + public void open(Configuration configuration) throws Exception { + FunctionUtils.openFunction(this.windowFunction, configuration); + } + + @Override + public void close() throws Exception { + FunctionUtils.closeFunction(this.windowFunction); + } + + @Override + public void setRuntimeContext(RuntimeContext t) { + super.setRuntimeContext(t); + + FunctionUtils.setFunctionRuntimeContext(this.windowFunction, t); + } + +} diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/RichProcessAllWindowFunction.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/RichProcessAllWindowFunction.java new file mode 100644 index 00000000000000..1130fa5ef86306 --- /dev/null +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/RichProcessAllWindowFunction.java @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.streaming.api.functions.windowing; + +import org.apache.flink.annotation.PublicEvolving; +import org.apache.flink.api.common.functions.IterationRuntimeContext; +import org.apache.flink.api.common.functions.RichFunction; +import org.apache.flink.api.common.functions.RuntimeContext; +import org.apache.flink.configuration.Configuration; +import org.apache.flink.streaming.api.windowing.windows.Window; + +/** + * Base rich abstract class for functions that are evaluated over keyed (grouped) windows using a context + * for passing extra information. + * + * @param The type of the input value. + * @param The type of the output value. + * @param The type of {@code Window} that this window function can be applied on. + */ +@PublicEvolving +public abstract class RichProcessAllWindowFunction + extends ProcessAllWindowFunction + implements RichFunction { + + private static final long serialVersionUID = 1L; + + + // -------------------------------------------------------------------------------------------- + // Runtime context access + // -------------------------------------------------------------------------------------------- + + private transient RuntimeContext runtimeContext; + + @Override + public void setRuntimeContext(RuntimeContext t) { + this.runtimeContext = t; + } + + @Override + public RuntimeContext getRuntimeContext() { + if (this.runtimeContext != null) { + return this.runtimeContext; + } else { + throw new IllegalStateException("The runtime context has not been initialized."); + } + } + + @Override + public IterationRuntimeContext getIterationRuntimeContext() { + if (this.runtimeContext == null) { + throw new IllegalStateException("The runtime context has not been initialized."); + } else if (this.runtimeContext instanceof IterationRuntimeContext) { + return (IterationRuntimeContext) this.runtimeContext; + } else { + throw new IllegalStateException("This stub is not part of an iteration step function."); + } + } + + // -------------------------------------------------------------------------------------------- + // Default life cycle methods + // -------------------------------------------------------------------------------------------- + + @Override + public void open(Configuration parameters) throws Exception {} + + @Override + public void close() throws Exception {} +} diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/functions/InternalAggregateProcessAllWindowFunction.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/functions/InternalAggregateProcessAllWindowFunction.java new file mode 100644 index 00000000000000..9533c95430c945 --- /dev/null +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/functions/InternalAggregateProcessAllWindowFunction.java @@ -0,0 +1,83 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.flink.streaming.runtime.operators.windowing.functions; + +import org.apache.flink.api.common.functions.AggregateFunction; +import org.apache.flink.api.common.functions.IterationRuntimeContext; +import org.apache.flink.api.common.functions.RuntimeContext; +import org.apache.flink.api.java.operators.translation.WrappingFunction; +import org.apache.flink.streaming.api.functions.windowing.ProcessAllWindowFunction; +import org.apache.flink.streaming.api.windowing.windows.Window; +import org.apache.flink.util.Collector; + +import java.util.Collections; + +/** + * Internal window function for wrapping a {@link ProcessAllWindowFunction} that takes an + * {@code Iterable} and an {@link AggregateFunction}. + * + * @param The window type + * @param The type of the input to the AggregateFunction + * @param The type of the AggregateFunction's accumulator + * @param The type of the AggregateFunction's result, and the input to the WindowFunction + * @param The result type of the WindowFunction + */ +public final class InternalAggregateProcessAllWindowFunction + extends WrappingFunction> + implements InternalWindowFunction, R, Byte, W> { + + private static final long serialVersionUID = 1L; + + private final AggregateFunction aggFunction; + + public InternalAggregateProcessAllWindowFunction( + AggregateFunction aggFunction, + ProcessAllWindowFunction windowFunction) { + super(windowFunction); + this.aggFunction = aggFunction; + } + + @Override + public void apply(Byte key, final W window, Iterable input, Collector out) throws Exception { + ProcessAllWindowFunction wrappedFunction = this.wrappedFunction; + ProcessAllWindowFunction.Context context = wrappedFunction.new Context() { + @Override + public W window() { + return window; + } + }; + + final ACC acc = aggFunction.createAccumulator(); + + for (T val : input) { + aggFunction.add(val, acc); + } + + wrappedFunction.process(context, Collections.singletonList(aggFunction.getResult(acc)), out); + } + + @Override + public RuntimeContext getRuntimeContext() { + throw new RuntimeException("This should never be called."); + } + + @Override + public IterationRuntimeContext getIterationRuntimeContext() { + throw new RuntimeException("This should never be called."); + } +} diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/functions/InternalIterableProcessAllWindowFunction.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/functions/InternalIterableProcessAllWindowFunction.java new file mode 100644 index 00000000000000..e33cc2aa89f1b3 --- /dev/null +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/functions/InternalIterableProcessAllWindowFunction.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.flink.streaming.runtime.operators.windowing.functions; + +import org.apache.flink.api.common.functions.IterationRuntimeContext; +import org.apache.flink.api.common.functions.RuntimeContext; +import org.apache.flink.api.java.operators.translation.WrappingFunction; +import org.apache.flink.streaming.api.functions.windowing.ProcessAllWindowFunction; +import org.apache.flink.streaming.api.windowing.windows.Window; +import org.apache.flink.util.Collector; + +/** + * Internal window function for wrapping a {@link ProcessAllWindowFunction} that takes an {@code Iterable} + * when the window state also is an {@code Iterable}. + */ +public final class InternalIterableProcessAllWindowFunction + extends WrappingFunction> + implements InternalWindowFunction, OUT, Byte, W> { + + private static final long serialVersionUID = 1L; + + public InternalIterableProcessAllWindowFunction(ProcessAllWindowFunction wrappedFunction) { + super(wrappedFunction); + } + + @Override + public void apply(Byte key, final W window, Iterable input, Collector out) throws Exception { + ProcessAllWindowFunction wrappedFunction = this.wrappedFunction; + ProcessAllWindowFunction.Context context = wrappedFunction.new Context() { + @Override + public W window() { + return window; + } + }; + + wrappedFunction.process(context, input, out); + } + + @Override + public RuntimeContext getRuntimeContext() { + throw new RuntimeException("This should never be called."); + } + + @Override + public IterationRuntimeContext getIterationRuntimeContext() { + throw new RuntimeException("This should never be called."); + } +} diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/functions/InternalSingleValueProcessAllWindowFunction.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/functions/InternalSingleValueProcessAllWindowFunction.java new file mode 100644 index 00000000000000..0284ef70feba7a --- /dev/null +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/functions/InternalSingleValueProcessAllWindowFunction.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.flink.streaming.runtime.operators.windowing.functions; + +import org.apache.flink.api.common.functions.IterationRuntimeContext; +import org.apache.flink.api.common.functions.RuntimeContext; +import org.apache.flink.api.java.operators.translation.WrappingFunction; +import org.apache.flink.streaming.api.functions.windowing.ProcessAllWindowFunction; +import org.apache.flink.streaming.api.windowing.windows.Window; +import org.apache.flink.util.Collector; + +import java.util.Collections; + +/** + * Internal window function for wrapping a {@link ProcessAllWindowFunction} that takes an {@code Iterable} + * when the window state is a single value. + */ +public final class InternalSingleValueProcessAllWindowFunction + extends WrappingFunction> + implements InternalWindowFunction { + + private static final long serialVersionUID = 1L; + + public InternalSingleValueProcessAllWindowFunction(ProcessAllWindowFunction wrappedFunction) { + super(wrappedFunction); + } + + @Override + public void apply(Byte key, final W window, IN input, Collector out) throws Exception { + ProcessAllWindowFunction wrappedFunction = this.wrappedFunction; + ProcessAllWindowFunction.Context context = wrappedFunction.new Context() { + @Override + public W window() { + return window; + } + }; + + wrappedFunction.process(context, Collections.singletonList(input), out); + } + + @Override + public RuntimeContext getRuntimeContext() { + throw new RuntimeException("This should never be called."); + } + + @Override + public IterationRuntimeContext getIterationRuntimeContext() { + throw new RuntimeException("This should never be called."); + } +} diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/functions/InternalSingleValueProcessWindowFunction.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/functions/InternalSingleValueProcessWindowFunction.java index b28c2082183b46..7a4e8c6ed7e0e6 100644 --- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/functions/InternalSingleValueProcessWindowFunction.java +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/functions/InternalSingleValueProcessWindowFunction.java @@ -21,14 +21,13 @@ import org.apache.flink.api.common.functions.RuntimeContext; import org.apache.flink.api.java.operators.translation.WrappingFunction; import org.apache.flink.streaming.api.functions.windowing.ProcessWindowFunction; -import org.apache.flink.streaming.api.functions.windowing.WindowFunction; import org.apache.flink.streaming.api.windowing.windows.Window; import org.apache.flink.util.Collector; import java.util.Collections; /** - * Internal window function for wrapping a {@link WindowFunction} that takes an {@code Iterable} + * Internal window function for wrapping a {@link ProcessWindowFunction} that takes an {@code Iterable} * when the window state is a single value. */ public final class InternalSingleValueProcessWindowFunction diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/FoldApplyProcessWindowFunctionTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/FoldApplyProcessWindowFunctionTest.java index af5c77a3e38e28..734879d5148209 100644 --- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/FoldApplyProcessWindowFunctionTest.java +++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/FoldApplyProcessWindowFunctionTest.java @@ -22,14 +22,17 @@ import org.apache.flink.api.common.functions.FoldFunction; import org.apache.flink.api.common.functions.util.ListCollector; import org.apache.flink.api.common.typeinfo.BasicTypeInfo; +import org.apache.flink.api.common.typeutils.base.ByteSerializer; import org.apache.flink.api.common.typeutils.base.IntSerializer; import org.apache.flink.api.java.Utils; import org.apache.flink.api.java.functions.KeySelector; import org.apache.flink.api.java.typeutils.TypeExtractor; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.flink.streaming.api.functions.source.SourceFunction; +import org.apache.flink.streaming.api.functions.windowing.FoldApplyProcessAllWindowFunction; import org.apache.flink.streaming.api.functions.windowing.FoldApplyProcessWindowFunction; import org.apache.flink.streaming.api.functions.windowing.FoldApplyWindowFunction; +import org.apache.flink.streaming.api.functions.windowing.ProcessAllWindowFunction; import org.apache.flink.streaming.api.functions.windowing.ProcessWindowFunction; import org.apache.flink.streaming.api.functions.windowing.WindowFunction; import org.apache.flink.streaming.api.graph.StreamGraph; @@ -39,6 +42,7 @@ import org.apache.flink.streaming.api.transformations.StreamTransformation; import org.apache.flink.streaming.api.windowing.windows.TimeWindow; import org.apache.flink.streaming.runtime.operators.windowing.AccumulatingProcessingTimeWindowOperator; +import org.apache.flink.streaming.runtime.operators.windowing.functions.InternalIterableProcessAllWindowFunction; import org.apache.flink.streaming.runtime.operators.windowing.functions.InternalIterableProcessWindowFunction; import org.apache.flink.util.Collector; import org.junit.Test; @@ -145,6 +149,101 @@ public TimeWindow window() { Assert.assertEquals(expected, result); } + /** + * Tests that the FoldWindowFunction gets the output type serializer set by the + * StreamGraphGenerator and checks that the FoldWindowFunction computes the correct result. + */ + @Test + public void testFoldAllWindowFunctionOutputTypeConfigurable() throws Exception{ + StreamExecutionEnvironment env = new DummyStreamExecutionEnvironment(); + + List> transformations = new ArrayList<>(); + + int initValue = 1; + + FoldApplyProcessAllWindowFunction foldWindowFunction = new FoldApplyProcessAllWindowFunction<>( + initValue, + new FoldFunction() { + @Override + public Integer fold(Integer accumulator, Integer value) throws Exception { + return accumulator + value; + } + + }, + new ProcessAllWindowFunction() { + @Override + public void process(Context context, + Iterable input, + Collector out) throws Exception { + for (Integer in: input) { + out.collect(in); + } + } + }, + BasicTypeInfo.INT_TYPE_INFO + ); + + AccumulatingProcessingTimeWindowOperator windowOperator = new AccumulatingProcessingTimeWindowOperator<>( + new InternalIterableProcessAllWindowFunction<>(foldWindowFunction), + new KeySelector() { + private static final long serialVersionUID = -7951310554369722809L; + + @Override + public Byte getKey(Integer value) throws Exception { + return 0; + } + }, + ByteSerializer.INSTANCE, + IntSerializer.INSTANCE, + 3000, + 3000 + ); + + SourceFunction sourceFunction = new SourceFunction(){ + + private static final long serialVersionUID = 8297735565464653028L; + + @Override + public void run(SourceContext ctx) throws Exception { + + } + + @Override + public void cancel() { + + } + }; + + SourceTransformation source = new SourceTransformation<>("", new StreamSource<>(sourceFunction), BasicTypeInfo.INT_TYPE_INFO, 1); + + transformations.add(new OneInputTransformation<>(source, "test", windowOperator, BasicTypeInfo.INT_TYPE_INFO, 1)); + + StreamGraph streamGraph = StreamGraphGenerator.generate(env, transformations); + + List result = new ArrayList<>(); + List input = new ArrayList<>(); + List expected = new ArrayList<>(); + + input.add(1); + input.add(2); + input.add(3); + + for (int value : input) { + initValue += value; + } + + expected.add(initValue); + + foldWindowFunction.process(foldWindowFunction.new Context() { + @Override + public TimeWindow window() { + return new TimeWindow(0, 1); + } + }, input, new ListCollector<>(result)); + + Assert.assertEquals(expected, result); + } + public static class DummyStreamExecutionEnvironment extends StreamExecutionEnvironment { @Override diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/StateDescriptorPassingTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/StateDescriptorPassingTest.java index 813ca9667aa8bb..f3062319ce6f24 100644 --- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/StateDescriptorPassingTest.java +++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/StateDescriptorPassingTest.java @@ -34,6 +34,7 @@ import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.flink.streaming.api.functions.windowing.AllWindowFunction; +import org.apache.flink.streaming.api.functions.windowing.ProcessAllWindowFunction; import org.apache.flink.streaming.api.functions.windowing.ProcessWindowFunction; import org.apache.flink.streaming.api.functions.windowing.WindowFunction; import org.apache.flink.streaming.api.transformations.OneInputTransformation; @@ -161,6 +162,24 @@ public void process(String s, Context ctx, validateListStateDescriptorConfigured(result); } + @Test + public void testProcessAllWindowState() throws Exception { + final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); + env.setStreamTimeCharacteristic(TimeCharacteristic.IngestionTime); + env.registerTypeWithKryoSerializer(File.class, JavaSerializer.class); + + DataStream src = env.fromElements(new File("/")); + + SingleOutputStreamOperator result = src + .timeWindowAll(Time.milliseconds(1000)) + .process(new ProcessAllWindowFunction() { + @Override + public void process(Context ctx, Iterable input, Collector out) {} + }); + + validateListStateDescriptorConfigured(result); + } + @Test public void testFoldWindowAllState() throws Exception { final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/windowing/functions/InternalWindowFunctionTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/windowing/functions/InternalWindowFunctionTest.java index e49a496870cde6..8f795e91364215 100644 --- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/windowing/functions/InternalWindowFunctionTest.java +++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/windowing/functions/InternalWindowFunctionTest.java @@ -26,15 +26,19 @@ import org.apache.flink.configuration.Configuration; import org.apache.flink.streaming.api.functions.util.StreamingFunctionUtils; import org.apache.flink.streaming.api.functions.windowing.RichAllWindowFunction; +import org.apache.flink.streaming.api.functions.windowing.RichProcessAllWindowFunction; import org.apache.flink.streaming.api.functions.windowing.RichProcessWindowFunction; import org.apache.flink.streaming.api.functions.windowing.RichWindowFunction; import org.apache.flink.streaming.api.operators.OutputTypeConfigurable; import org.apache.flink.streaming.api.windowing.windows.TimeWindow; +import org.apache.flink.streaming.runtime.operators.windowing.functions.InternalAggregateProcessAllWindowFunction; import org.apache.flink.streaming.runtime.operators.windowing.functions.InternalAggregateProcessWindowFunction; import org.apache.flink.streaming.runtime.operators.windowing.functions.InternalIterableAllWindowFunction; +import org.apache.flink.streaming.runtime.operators.windowing.functions.InternalIterableProcessAllWindowFunction; import org.apache.flink.streaming.runtime.operators.windowing.functions.InternalIterableProcessWindowFunction; import org.apache.flink.streaming.runtime.operators.windowing.functions.InternalIterableWindowFunction; import org.apache.flink.streaming.runtime.operators.windowing.functions.InternalSingleValueAllWindowFunction; +import org.apache.flink.streaming.runtime.operators.windowing.functions.InternalSingleValueProcessAllWindowFunction; import org.apache.flink.streaming.runtime.operators.windowing.functions.InternalSingleValueProcessWindowFunction; import org.apache.flink.streaming.runtime.operators.windowing.functions.InternalSingleValueWindowFunction; import org.apache.flink.util.Collector; @@ -97,6 +101,47 @@ public void testInternalIterableAllWindowFunction() throws Exception { verify(mock).close(); } + @SuppressWarnings("unchecked") + @Test + public void testInternalIterableProcessAllWindowFunction() throws Exception { + + ProcessAllWindowFunctionMock mock = mock(ProcessAllWindowFunctionMock.class); + InternalIterableProcessAllWindowFunction windowFunction = + new InternalIterableProcessAllWindowFunction<>(mock); + + // check setOutputType + TypeInformation stringType = BasicTypeInfo.STRING_TYPE_INFO; + ExecutionConfig execConf = new ExecutionConfig(); + execConf.setParallelism(42); + + StreamingFunctionUtils.setOutputType(windowFunction, stringType, execConf); + verify(mock).setOutputType(stringType, execConf); + + // check open + Configuration config = new Configuration(); + + windowFunction.open(config); + verify(mock).open(config); + + // check setRuntimeContext + RuntimeContext rCtx = mock(RuntimeContext.class); + + windowFunction.setRuntimeContext(rCtx); + verify(mock).setRuntimeContext(rCtx); + + // check apply + TimeWindow w = mock(TimeWindow.class); + Iterable i = (Iterable)mock(Iterable.class); + Collector c = (Collector) mock(Collector.class); + + windowFunction.apply(((byte)0), w, i, c); + verify(mock).process((ProcessAllWindowFunctionMock.Context) anyObject(), eq(i), eq(c)); + + // check close + windowFunction.close(); + verify(mock).close(); + } + @SuppressWarnings("unchecked") @Test public void testInternalIterableWindowFunction() throws Exception { @@ -261,6 +306,47 @@ public void testInternalSingleValueAllWindowFunction() throws Exception { verify(mock).close(); } + @SuppressWarnings("unchecked") + @Test + public void testInternalSingleValueProcessAllWindowFunction() throws Exception { + + ProcessAllWindowFunctionMock mock = mock(ProcessAllWindowFunctionMock.class); + InternalSingleValueProcessAllWindowFunction windowFunction = + new InternalSingleValueProcessAllWindowFunction<>(mock); + + // check setOutputType + TypeInformation stringType = BasicTypeInfo.STRING_TYPE_INFO; + ExecutionConfig execConf = new ExecutionConfig(); + execConf.setParallelism(42); + + StreamingFunctionUtils.setOutputType(windowFunction, stringType, execConf); + + verify(mock).setOutputType(stringType, execConf); + + // check open + Configuration config = new Configuration(); + + windowFunction.open(config); + verify(mock).open(config); + + // check setRuntimeContext + RuntimeContext rCtx = mock(RuntimeContext.class); + + windowFunction.setRuntimeContext(rCtx); + verify(mock).setRuntimeContext(rCtx); + + // check apply + TimeWindow w = mock(TimeWindow.class); + Collector c = (Collector) mock(Collector.class); + + windowFunction.apply(((byte)0), w, 23L, c); + verify(mock).process((ProcessAllWindowFunctionMock.Context) anyObject(), (Iterable)argThat(IsIterableContainingInOrder.contains(23L)), eq(c)); + + // check close + windowFunction.close(); + verify(mock).close(); + } + @SuppressWarnings("unchecked") @Test public void testInternalSingleValueProcessWindowFunction() throws Exception { @@ -310,7 +396,7 @@ public void testInternalAggregateProcessWindowFunction() throws Exception { InternalAggregateProcessWindowFunction, Map, String, Long, TimeWindow> windowFunction = new InternalAggregateProcessWindowFunction<>(new AggregateFunction, Map>() { private static final long serialVersionUID = 1L; - + @Override public Set createAccumulator() { return new HashSet<>(); @@ -364,7 +450,7 @@ public Set merge(Set a, Set b) { List args = new LinkedList<>(); args.add(23L); args.add(24L); - + windowFunction.apply(42L, w, args, c); verify(mock).process( eq(42L), @@ -379,6 +465,83 @@ public Set merge(Set a, Set b) { verify(mock).close(); } + @SuppressWarnings("unchecked") + @Test + public void testInternalAggregateProcessAllWindowFunction() throws Exception { + + AggregateProcessAllWindowFunctionMock mock = mock(AggregateProcessAllWindowFunctionMock.class); + + InternalAggregateProcessAllWindowFunction, Map, String, TimeWindow> windowFunction = + new InternalAggregateProcessAllWindowFunction<>(new AggregateFunction, Map>() { + private static final long serialVersionUID = 1L; + + @Override + public Set createAccumulator() { + return new HashSet<>(); + } + + @Override + public void add(Long value, Set accumulator) { + accumulator.add(value); + } + + @Override + public Map getResult(Set accumulator) { + Map result = new HashMap<>(); + for (Long in : accumulator) { + result.put(in, in); + } + return result; + } + + @Override + public Set merge(Set a, Set b) { + a.addAll(b); + return a; + } + }, mock); + + // check setOutputType + TypeInformation stringType = BasicTypeInfo.STRING_TYPE_INFO; + ExecutionConfig execConf = new ExecutionConfig(); + execConf.setParallelism(42); + + StreamingFunctionUtils.setOutputType(windowFunction, stringType, execConf); + verify(mock).setOutputType(stringType, execConf); + + // check open + Configuration config = new Configuration(); + + windowFunction.open(config); + verify(mock).open(config); + + // check setRuntimeContext + RuntimeContext rCtx = mock(RuntimeContext.class); + + windowFunction.setRuntimeContext(rCtx); + verify(mock).setRuntimeContext(rCtx); + + // check apply + TimeWindow w = mock(TimeWindow.class); + Collector c = (Collector) mock(Collector.class); + + List args = new LinkedList<>(); + args.add(23L); + args.add(24L); + + windowFunction.apply(((byte)0), w, args, c); + verify(mock).process( + (AggregateProcessAllWindowFunctionMock.Context) anyObject(), + (Iterable) argThat(containsInAnyOrder(allOf( + hasEntry(is(23L), is(23L)), + hasEntry(is(24L), is(24L))))), + eq(c)); + + // check close + windowFunction.close(); + verify(mock).close(); + } + public static class ProcessWindowFunctionMock extends RichProcessWindowFunction implements OutputTypeConfigurable { @@ -405,6 +568,19 @@ public void setOutputType(TypeInformation outTypeInfo, ExecutionConfig e public void process(Long aLong, Context context, Iterable> input, Collector out) throws Exception { } } + public static class AggregateProcessAllWindowFunctionMock + extends RichProcessAllWindowFunction, String, TimeWindow> + implements OutputTypeConfigurable { + + private static final long serialVersionUID = 1L; + + @Override + public void setOutputType(TypeInformation outTypeInfo, ExecutionConfig executionConfig) { } + + @Override + public void process(Context context, Iterable> input, Collector out) throws Exception { } + } + public static class WindowFunctionMock extends RichWindowFunction implements OutputTypeConfigurable { @@ -430,4 +606,17 @@ public void setOutputType(TypeInformation outTypeInfo, ExecutionConfig e @Override public void apply(TimeWindow window, Iterable values, Collector out) throws Exception { } } + + public static class ProcessAllWindowFunctionMock + extends RichProcessAllWindowFunction + implements OutputTypeConfigurable { + + private static final long serialVersionUID = 1L; + + @Override + public void setOutputType(TypeInformation outTypeInfo, ExecutionConfig executionConfig) { } + + @Override + public void process(Context context, Iterable input, Collector out) throws Exception { } + } } diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/AllWindowTranslationTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/AllWindowTranslationTest.java index b6c161827d6513..34eac9e384c84a 100644 --- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/AllWindowTranslationTest.java +++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/AllWindowTranslationTest.java @@ -38,6 +38,7 @@ import org.apache.flink.streaming.api.datastream.DataStream; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.flink.streaming.api.functions.windowing.AllWindowFunction; +import org.apache.flink.streaming.api.functions.windowing.ProcessAllWindowFunction; import org.apache.flink.streaming.api.operators.OneInputStreamOperator; import org.apache.flink.streaming.api.transformations.OneInputTransformation; import org.apache.flink.streaming.api.windowing.assigners.EventTimeSessionWindows; @@ -383,6 +384,119 @@ public void apply( processElementAndEnsureOutput(operator, winOperator.getKeySelector(), BasicTypeInfo.STRING_TYPE_INFO, new Tuple2<>("hello", 1)); } + @Test + @SuppressWarnings("rawtypes") + public void testReduceWithProcessWindowFunctionEventTime() throws Exception { + StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); + env.setStreamTimeCharacteristic(TimeCharacteristic.IngestionTime); + + DataStream> source = env.fromElements(Tuple2.of("hello", 1), Tuple2.of("hello", 2)); + + DummyReducer reducer = new DummyReducer(); + + DataStream> window = source + .windowAll(TumblingEventTimeWindows.of(Time.of(1, TimeUnit.SECONDS))) + .reduce(reducer, new ProcessAllWindowFunction, Tuple3, TimeWindow>() { + private static final long serialVersionUID = 1L; + + @Override + public void process( + Context ctx, + Iterable> values, + Collector> out) throws Exception { + for (Tuple2 in : values) { + out.collect(new Tuple3<>(in.f0, in.f0, in.f1)); + } + } + }); + + OneInputTransformation, Tuple3> transform = + (OneInputTransformation, Tuple3>) window.getTransformation(); + OneInputStreamOperator, Tuple3> operator = transform.getOperator(); + Assert.assertTrue(operator instanceof WindowOperator); + WindowOperator, ?, ?, ?> winOperator = (WindowOperator, ?, ?, ?>) operator; + Assert.assertTrue(winOperator.getTrigger() instanceof EventTimeTrigger); + Assert.assertTrue(winOperator.getWindowAssigner() instanceof TumblingEventTimeWindows); + Assert.assertTrue(winOperator.getStateDescriptor() instanceof ReducingStateDescriptor); + + processElementAndEnsureOutput(operator, winOperator.getKeySelector(), BasicTypeInfo.STRING_TYPE_INFO, new Tuple2<>("hello", 1)); + } + + @Test + @SuppressWarnings("rawtypes") + public void testReduceWithProcessWindowFunctionProcessingTime() throws Exception { + StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); + env.setStreamTimeCharacteristic(TimeCharacteristic.ProcessingTime); + + DataStream> source = env.fromElements(Tuple2.of("hello", 1), Tuple2.of("hello", 2)); + + DataStream> window = source + .windowAll(TumblingProcessingTimeWindows.of(Time.of(1, TimeUnit.SECONDS))) + .reduce(new DummyReducer(), new ProcessAllWindowFunction, Tuple3, TimeWindow>() { + private static final long serialVersionUID = 1L; + + @Override + public void process( + Context ctx, + Iterable> values, + Collector> out) throws Exception { + for (Tuple2 in : values) { + out.collect(new Tuple3<>(in.f0, in.f0, in.f1)); + } + } + }); + + OneInputTransformation, Tuple3> transform = + (OneInputTransformation, Tuple3>) window.getTransformation(); + OneInputStreamOperator, Tuple3> operator = transform.getOperator(); + Assert.assertTrue(operator instanceof WindowOperator); + WindowOperator, ?, ?, ?> winOperator = (WindowOperator, ?, ?, ?>) operator; + Assert.assertTrue(winOperator.getTrigger() instanceof ProcessingTimeTrigger); + Assert.assertTrue(winOperator.getWindowAssigner() instanceof TumblingProcessingTimeWindows); + Assert.assertTrue(winOperator.getStateDescriptor() instanceof ReducingStateDescriptor); + + processElementAndEnsureOutput(operator, winOperator.getKeySelector(), BasicTypeInfo.STRING_TYPE_INFO, new Tuple2<>("hello", 1)); + } + + @Test + @SuppressWarnings("rawtypes") + public void testReduceWithEvictorAndProcessFunction() throws Exception { + StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); + env.setStreamTimeCharacteristic(TimeCharacteristic.IngestionTime); + + DataStream> source = env.fromElements(Tuple2.of("hello", 1), Tuple2.of("hello", 2)); + + DummyReducer reducer = new DummyReducer(); + + DataStream> window1 = source + .windowAll(SlidingEventTimeWindows.of(Time.of(1, TimeUnit.SECONDS), Time.of(100, TimeUnit.MILLISECONDS))) + .evictor(CountEvictor.of(100)) + .reduce( + reducer, + new ProcessAllWindowFunction, Tuple2, TimeWindow>() { + @Override + public void process( + Context context, + Iterable> elements, + Collector> out) throws Exception { + for (Tuple2 in : elements) { + out.collect(in); + } + } + }); + + OneInputTransformation, Tuple2> transform = (OneInputTransformation, Tuple2>) window1.getTransformation(); + OneInputStreamOperator, Tuple2> operator = transform.getOperator(); + Assert.assertTrue(operator instanceof EvictingWindowOperator); + EvictingWindowOperator, ?, ?> winOperator = (EvictingWindowOperator, ?, ?>) operator; + Assert.assertTrue(winOperator.getTrigger() instanceof EventTimeTrigger); + Assert.assertTrue(winOperator.getEvictor() instanceof CountEvictor); + Assert.assertTrue(winOperator.getWindowAssigner() instanceof SlidingEventTimeWindows); + Assert.assertTrue(winOperator.getStateDescriptor() instanceof ListStateDescriptor); + + processElementAndEnsureOutput(winOperator, winOperator.getKeySelector(), BasicTypeInfo.STRING_TYPE_INFO, new Tuple2<>("hello", 1)); + } + /** * Test for the deprecated .apply(Reducer, WindowFunction). */ @@ -540,6 +654,226 @@ public void testAggregateWithWindowFunctionProcessingTime() throws Exception { operator, winOperator.getKeySelector(), BasicTypeInfo.STRING_TYPE_INFO, new Tuple2<>("hello", 1)); } + @Test + public void testAggregateWithEvictor() throws Exception { + StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); + env.setStreamTimeCharacteristic(TimeCharacteristic.IngestionTime); + + DataStream> source = env.fromElements(Tuple2.of("hello", 1), Tuple2.of("hello", 2)); + + DataStream> window1 = source + .windowAll(SlidingEventTimeWindows.of(Time.of(1, TimeUnit.SECONDS), Time.of(100, TimeUnit.MILLISECONDS))) + .evictor(CountEvictor.of(100)) + .aggregate(new DummyAggregationFunction()); + + OneInputTransformation, Tuple2> transform = + (OneInputTransformation, Tuple2>) window1.getTransformation(); + + OneInputStreamOperator, Tuple2> operator = transform.getOperator(); + + Assert.assertTrue(operator instanceof WindowOperator); + WindowOperator, ?, ?, ?> winOperator = + (WindowOperator, ?, ?, ?>) operator; + + Assert.assertTrue(winOperator.getTrigger() instanceof EventTimeTrigger); + Assert.assertTrue(winOperator.getWindowAssigner() instanceof SlidingEventTimeWindows); + Assert.assertTrue(winOperator.getStateDescriptor() instanceof ListStateDescriptor); + + processElementAndEnsureOutput( + winOperator, winOperator.getKeySelector(), BasicTypeInfo.STRING_TYPE_INFO, new Tuple2<>("hello", 1)); + } + + @Test + public void testAggregateWithEvictorAndProcessFunction() throws Exception { + StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); + env.setStreamTimeCharacteristic(TimeCharacteristic.IngestionTime); + + DataStream> source = env.fromElements(Tuple2.of("hello", 1), Tuple2.of("hello", 2)); + + DataStream> window1 = source + .windowAll(SlidingEventTimeWindows.of(Time.of(1, TimeUnit.SECONDS), Time.of(100, TimeUnit.MILLISECONDS))) + .evictor(CountEvictor.of(100)) + .aggregate( + new DummyAggregationFunction(), + new ProcessAllWindowFunction, Tuple2, TimeWindow>() { + @Override + public void process( + Context context, + Iterable> elements, + Collector> out) throws Exception { + for (Tuple2 in : elements) { + out.collect(in); + } + } + }); + + OneInputTransformation, Tuple2> transform = + (OneInputTransformation, Tuple2>) window1.getTransformation(); + + OneInputStreamOperator, Tuple2> operator = transform.getOperator(); + + Assert.assertTrue(operator instanceof WindowOperator); + WindowOperator, ?, ?, ?> winOperator = + (WindowOperator, ?, ?, ?>) operator; + + Assert.assertTrue(winOperator.getTrigger() instanceof EventTimeTrigger); + Assert.assertTrue(winOperator.getWindowAssigner() instanceof SlidingEventTimeWindows); + Assert.assertTrue(winOperator.getStateDescriptor() instanceof ListStateDescriptor); + + processElementAndEnsureOutput( + winOperator, winOperator.getKeySelector(), BasicTypeInfo.STRING_TYPE_INFO, new Tuple2<>("hello", 1)); + } + + // ------------------------------------------------------------------------ + // process() translation tests + // ------------------------------------------------------------------------ + + @Test + @SuppressWarnings("rawtypes") + public void testProcessEventTime() throws Exception { + StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); + env.setStreamTimeCharacteristic(TimeCharacteristic.IngestionTime); + + DataStream> source = env.fromElements(Tuple2.of("hello", 1), Tuple2.of("hello", 2)); + + DataStream> window1 = source + .windowAll(TumblingEventTimeWindows.of(Time.of(1, TimeUnit.SECONDS))) + .process(new ProcessAllWindowFunction, Tuple2, TimeWindow>() { + private static final long serialVersionUID = 1L; + + @Override + public void process( + Context ctx, + Iterable> values, + Collector> out) throws Exception { + for (Tuple2 in : values) { + out.collect(in); + } + } + }); + + OneInputTransformation, Tuple2> transform = (OneInputTransformation, Tuple2>) window1.getTransformation(); + OneInputStreamOperator, Tuple2> operator = transform.getOperator(); + Assert.assertTrue(operator instanceof WindowOperator); + WindowOperator, ?, ?, ?> winOperator = (WindowOperator, ?, ?, ?>) operator; + Assert.assertTrue(winOperator.getTrigger() instanceof EventTimeTrigger); + Assert.assertTrue(winOperator.getWindowAssigner() instanceof TumblingEventTimeWindows); + Assert.assertTrue(winOperator.getStateDescriptor() instanceof ListStateDescriptor); + + processElementAndEnsureOutput(winOperator, winOperator.getKeySelector(), BasicTypeInfo.STRING_TYPE_INFO, new Tuple2<>("hello", 1)); + } + + @Test + @SuppressWarnings("rawtypes") + public void testProcessProcessingTime() throws Exception { + StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); + env.setStreamTimeCharacteristic(TimeCharacteristic.ProcessingTime); + + DataStream> source = env.fromElements(Tuple2.of("hello", 1), Tuple2.of("hello", 2)); + + DataStream> window1 = source + .windowAll(TumblingProcessingTimeWindows.of(Time.of(1, TimeUnit.SECONDS))) + .process(new ProcessAllWindowFunction, Tuple2, TimeWindow>() { + private static final long serialVersionUID = 1L; + + @Override + public void process( + Context ctx, + Iterable> values, + Collector> out) throws Exception { + for (Tuple2 in : values) { + out.collect(in); + } + } + }); + + OneInputTransformation, Tuple2> transform = (OneInputTransformation, Tuple2>) window1.getTransformation(); + OneInputStreamOperator, Tuple2> operator = transform.getOperator(); + Assert.assertTrue(operator instanceof WindowOperator); + WindowOperator, ?, ?, ?> winOperator = (WindowOperator, ?, ?, ?>) operator; + Assert.assertTrue(winOperator.getTrigger() instanceof ProcessingTimeTrigger); + Assert.assertTrue(winOperator.getWindowAssigner() instanceof TumblingProcessingTimeWindows); + Assert.assertTrue(winOperator.getStateDescriptor() instanceof ListStateDescriptor); + + processElementAndEnsureOutput(winOperator, winOperator.getKeySelector(), BasicTypeInfo.STRING_TYPE_INFO, new Tuple2<>("hello", 1)); + + } + + @Test + @SuppressWarnings("rawtypes") + public void testProcessWithEvictor() throws Exception { + StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); + env.setStreamTimeCharacteristic(TimeCharacteristic.IngestionTime); + + DataStream> source = env.fromElements(Tuple2.of("hello", 1), Tuple2.of("hello", 2)); + + DataStream> window1 = source + .windowAll(TumblingEventTimeWindows.of(Time.of(1, TimeUnit.SECONDS))) + .trigger(CountTrigger.of(1)) + .evictor(TimeEvictor.of(Time.of(100, TimeUnit.MILLISECONDS))) + .process(new ProcessAllWindowFunction, Tuple2, TimeWindow>() { + private static final long serialVersionUID = 1L; + + @Override + public void process( + Context ctx, + Iterable> values, + Collector> out) throws Exception { + for (Tuple2 in : values) { + out.collect(in); + } + } + }); + + OneInputTransformation, Tuple2> transform = (OneInputTransformation, Tuple2>) window1.getTransformation(); + OneInputStreamOperator, Tuple2> operator = transform.getOperator(); + Assert.assertTrue(operator instanceof EvictingWindowOperator); + EvictingWindowOperator, ?, ?> winOperator = (EvictingWindowOperator, ?, ?>) operator; + Assert.assertTrue(winOperator.getTrigger() instanceof CountTrigger); + Assert.assertTrue(winOperator.getEvictor() instanceof TimeEvictor); + Assert.assertTrue(winOperator.getWindowAssigner() instanceof TumblingEventTimeWindows); + Assert.assertTrue(winOperator.getStateDescriptor() instanceof ListStateDescriptor); + + processElementAndEnsureOutput(winOperator, winOperator.getKeySelector(), BasicTypeInfo.STRING_TYPE_INFO, new Tuple2<>("hello", 1)); + } + + @Test + @SuppressWarnings("rawtypes") + public void testProcessWithCustomTrigger() throws Exception { + StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); + env.setStreamTimeCharacteristic(TimeCharacteristic.IngestionTime); + + DataStream> source = env.fromElements(Tuple2.of("hello", 1), Tuple2.of("hello", 2)); + + DataStream> window1 = source + .windowAll(TumblingEventTimeWindows.of(Time.of(1, TimeUnit.SECONDS))) + .trigger(CountTrigger.of(1)) + .process(new ProcessAllWindowFunction, Tuple2, TimeWindow>() { + private static final long serialVersionUID = 1L; + + @Override + public void process( + Context ctx, + Iterable> values, + Collector> out) throws Exception { + for (Tuple2 in : values) { + out.collect(in); + } + } + }); + + OneInputTransformation, Tuple2> transform = (OneInputTransformation, Tuple2>) window1.getTransformation(); + OneInputStreamOperator, Tuple2> operator = transform.getOperator(); + Assert.assertTrue(operator instanceof WindowOperator); + WindowOperator, ?, ?, ?> winOperator = (WindowOperator, ?, ?, ?>) operator; + Assert.assertTrue(winOperator.getTrigger() instanceof CountTrigger); + Assert.assertTrue(winOperator.getWindowAssigner() instanceof TumblingEventTimeWindows); + Assert.assertTrue(winOperator.getStateDescriptor() instanceof ListStateDescriptor); + + processElementAndEnsureOutput(winOperator, winOperator.getKeySelector(), BasicTypeInfo.STRING_TYPE_INFO, new Tuple2<>("hello", 1)); + } + + // ------------------------------------------------------------------------ // fold() translation tests // ------------------------------------------------------------------------ @@ -664,6 +998,117 @@ public void apply( processElementAndEnsureOutput(winOperator, winOperator.getKeySelector(), BasicTypeInfo.STRING_TYPE_INFO, new Tuple2<>("hello", 1)); } + @Test + @SuppressWarnings("rawtypes") + public void testFoldWithProcessAllWindowFunctionEventTime() throws Exception { + StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); + env.setStreamTimeCharacteristic(TimeCharacteristic.IngestionTime); + + DataStream> source = env.fromElements(Tuple2.of("hello", 1), Tuple2.of("hello", 2)); + + DataStream> window = source + .windowAll(TumblingEventTimeWindows.of(Time.of(1, TimeUnit.SECONDS))) + .fold(new Tuple3<>("", "", 0), new DummyFolder(), new ProcessAllWindowFunction, Tuple2, TimeWindow>() { + private static final long serialVersionUID = 1L; + @Override + public void process( + Context ctx, + Iterable> values, + Collector> out) throws Exception { + for (Tuple3 in : values) { + out.collect(new Tuple2<>(in.f0, in.f2)); + } + } + }); + + OneInputTransformation, Tuple2> transform = + (OneInputTransformation, Tuple2>) window.getTransformation(); + OneInputStreamOperator, Tuple2> operator = transform.getOperator(); + Assert.assertTrue(operator instanceof WindowOperator); + WindowOperator, ?, ?, ?> winOperator = (WindowOperator, ?, ?, ?>) operator; + Assert.assertTrue(winOperator.getTrigger() instanceof EventTimeTrigger); + Assert.assertTrue(winOperator.getWindowAssigner() instanceof TumblingEventTimeWindows); + Assert.assertTrue(winOperator.getStateDescriptor() instanceof FoldingStateDescriptor); + + processElementAndEnsureOutput(winOperator, winOperator.getKeySelector(), BasicTypeInfo.STRING_TYPE_INFO, new Tuple2<>("hello", 1)); + } + + @Test + @SuppressWarnings("rawtypes") + public void testFoldWithProcessAllWindowFunctionProcessingTime() throws Exception { + StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); + env.setStreamTimeCharacteristic(TimeCharacteristic.ProcessingTime); + + DataStream> source = env.fromElements(Tuple2.of("hello", 1), Tuple2.of("hello", 2)); + + DataStream> window = source + .windowAll(TumblingProcessingTimeWindows.of(Time.of(1, TimeUnit.SECONDS))) + .fold(new Tuple3<>("", "empty", 0), new DummyFolder(), new ProcessAllWindowFunction, Tuple2, TimeWindow>() { + private static final long serialVersionUID = 1L; + + @Override + public void process( + Context ctx, + Iterable> values, + Collector> out) throws Exception { + for (Tuple3 in : values) { + out.collect(new Tuple2<>(in.f0, in.f2)); + } + } + }); + + OneInputTransformation, Tuple2> transform = + (OneInputTransformation, Tuple2>) window.getTransformation(); + OneInputStreamOperator, Tuple2> operator = transform.getOperator(); + Assert.assertTrue(operator instanceof WindowOperator); + WindowOperator, ?, ?, ?> winOperator = (WindowOperator, ?, ?, ?>) operator; + Assert.assertTrue(winOperator.getTrigger() instanceof ProcessingTimeTrigger); + Assert.assertTrue(winOperator.getWindowAssigner() instanceof TumblingProcessingTimeWindows); + Assert.assertTrue(winOperator.getStateDescriptor() instanceof FoldingStateDescriptor); + + processElementAndEnsureOutput(winOperator, winOperator.getKeySelector(), BasicTypeInfo.STRING_TYPE_INFO, new Tuple2<>("hello", 1)); + } + + @Test + @SuppressWarnings({"rawtypes", "unchecked"}) + public void testFoldWithEvictorAndProcessFunction() throws Exception { + StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); + env.setStreamTimeCharacteristic(TimeCharacteristic.IngestionTime); + + DataStream> source = env.fromElements(Tuple2.of("hello", 1), Tuple2.of("hello", 2)); + + DataStream> window1 = source + .windowAll(SlidingEventTimeWindows.of(Time.of(1, TimeUnit.SECONDS), Time.of(100, TimeUnit.MILLISECONDS))) + .evictor(CountEvictor.of(100)) + .fold( + new Tuple3<>("", "", 1), + new DummyFolder(), + new ProcessAllWindowFunction, Tuple3, TimeWindow>() { + @Override + public void process( + Context context, + Iterable> elements, + Collector> out) throws Exception { + for (Tuple3 in : elements) { + out.collect(in); + } + } + }); + + OneInputTransformation, Tuple3> transform = + (OneInputTransformation, Tuple3>) window1.getTransformation(); + OneInputStreamOperator, Tuple3> operator = transform.getOperator(); + Assert.assertTrue(operator instanceof EvictingWindowOperator); + EvictingWindowOperator, ?, ?> winOperator = (EvictingWindowOperator, ?, ?>) operator; + Assert.assertTrue(winOperator.getTrigger() instanceof EventTimeTrigger); + Assert.assertTrue(winOperator.getEvictor() instanceof CountEvictor); + Assert.assertTrue(winOperator.getWindowAssigner() instanceof SlidingEventTimeWindows); + Assert.assertTrue(winOperator.getStateDescriptor() instanceof ListStateDescriptor); + + winOperator.setOutputType((TypeInformation) window1.getType(), new ExecutionConfig()); + processElementAndEnsureOutput(winOperator, winOperator.getKeySelector(), BasicTypeInfo.STRING_TYPE_INFO, new Tuple2<>("hello", 1)); + } + @Test @SuppressWarnings("rawtypes") public void testApplyWithPreFolderEventTime() throws Exception { diff --git a/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/AllWindowedStream.scala b/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/AllWindowedStream.scala index 7f52252a87ad5e..4d31ddeb2f6642 100644 --- a/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/AllWindowedStream.scala +++ b/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/AllWindowedStream.scala @@ -18,20 +18,19 @@ package org.apache.flink.streaming.api.scala -import org.apache.flink.annotation.{PublicEvolving, Public} +import org.apache.flink.annotation.{Public, PublicEvolving} import org.apache.flink.api.common.functions.{AggregateFunction, FoldFunction, ReduceFunction} import org.apache.flink.api.common.typeinfo.TypeInformation import org.apache.flink.streaming.api.datastream.{AllWindowedStream => JavaAllWStream} import org.apache.flink.streaming.api.functions.aggregation.AggregationFunction.AggregationType import org.apache.flink.streaming.api.functions.aggregation.{ComparableAggregator, SumAggregator} -import org.apache.flink.streaming.api.scala.function.AllWindowFunction -import org.apache.flink.streaming.api.scala.function.util.{ScalaAllWindowFunction, ScalaAllWindowFunctionWrapper, ScalaReduceFunction, ScalaFoldFunction} +import org.apache.flink.streaming.api.scala.function.{AllWindowFunction, ProcessAllWindowFunction} +import org.apache.flink.streaming.api.scala.function.util.{ScalaAllWindowFunction, ScalaAllWindowFunctionWrapper, ScalaFoldFunction, ScalaProcessAllWindowFunctionWrapper, ScalaReduceFunction} import org.apache.flink.streaming.api.windowing.evictors.Evictor import org.apache.flink.streaming.api.windowing.time.Time import org.apache.flink.streaming.api.windowing.triggers.Trigger import org.apache.flink.streaming.api.windowing.windows.Window import org.apache.flink.util.Collector - import org.apache.flink.util.Preconditions.checkNotNull /** @@ -199,6 +198,62 @@ class AllWindowedStream[T, W <: Window](javaStream: JavaAllWStream[T, W]) { asScalaStream(javaStream.reduce(reducer, applyFunction, returnType)) } + /** + * Applies the given window function to each window. The window function is called for each + * evaluation of the window for each key individually. The output of the window function is + * interpreted as a regular non-windowed stream. + * + * Arriving data is pre-aggregated using the given pre-aggregation reducer. + * + * @param preAggregator The reduce function that is used for pre-aggregation + * @param windowFunction The process window function. + * @return The data stream that is the result of applying the window function to the window. + */ + def reduce[R: TypeInformation]( + preAggregator: ReduceFunction[T], + windowFunction: ProcessAllWindowFunction[T, R, W]): DataStream[R] = { + + val cleanedReducer = clean(preAggregator) + val cleanedWindowFunction = clean(windowFunction) + + val applyFunction = new ScalaProcessAllWindowFunctionWrapper[T, R, W](cleanedWindowFunction) + + val returnType: TypeInformation[R] = implicitly[TypeInformation[R]] + asScalaStream(javaStream.reduce(cleanedReducer, applyFunction, returnType)) + } + + /** + * Applies the given window function to each window. The window function is called for each + * evaluation of the window for each key individually. The output of the window function is + * interpreted as a regular non-windowed stream. + * + * Arriving data is pre-aggregated using the given pre-aggregation reducer. + * + * @param preAggregator The reduce function that is used for pre-aggregation + * @param windowFunction The process window function. + * @return The data stream that is the result of applying the window function to the window. + */ + def reduce[R: TypeInformation]( + preAggregator: (T, T) => T, + windowFunction: ProcessAllWindowFunction[T, R, W]): DataStream[R] = { + + if (preAggregator == null) { + throw new NullPointerException("Reduce function must not be null.") + } + if (windowFunction == null) { + throw new NullPointerException("WindowApply function must not be null.") + } + + val cleanReducer = clean(preAggregator) + val cleanWindowFunction = clean(windowFunction) + + val reducer = new ScalaReduceFunction[T](cleanReducer) + val applyFunction = new ScalaProcessAllWindowFunctionWrapper[T, R, W](cleanWindowFunction) + + val returnType: TypeInformation[R] = implicitly[TypeInformation[R]] + asScalaStream(javaStream.reduce(reducer, applyFunction, returnType)) + } + // --------------------------- aggregate() ---------------------------------- /** @@ -253,6 +308,38 @@ class AllWindowedStream[T, W <: Window](javaStream: JavaAllWStream[T, W]) { accumulatorType, aggregationResultType, resultType)) } + /** + * Applies the given window function to each window. The window function is called for each + * evaluation of the window for each key individually. The output of the window function is + * interpreted as a regular non-windowed stream. + * + * Arriving data is pre-aggregated using the given aggregation function. + * + * @param preAggregator The aggregation function that is used for pre-aggregation + * @param windowFunction The process window function. + * @return The data stream that is the result of applying the window function to the window. + */ + def aggregate[ACC: TypeInformation, V: TypeInformation, R: TypeInformation] + (preAggregator: AggregateFunction[T, ACC, V], + windowFunction: ProcessAllWindowFunction[V, R, W]): DataStream[R] = { + + checkNotNull(preAggregator, "AggregationFunction must not be null") + checkNotNull(windowFunction, "Window function must not be null") + + val cleanedPreAggregator = clean(preAggregator) + val cleanedWindowFunction = clean(windowFunction) + + val applyFunction = new ScalaProcessAllWindowFunctionWrapper[V, R, W](cleanedWindowFunction) + + val accumulatorType: TypeInformation[ACC] = implicitly[TypeInformation[ACC]] + val aggregationResultType: TypeInformation[V] = implicitly[TypeInformation[V]] + val resultType: TypeInformation[R] = implicitly[TypeInformation[R]] + + asScalaStream(javaStream.aggregate( + cleanedPreAggregator, applyFunction, + accumulatorType, aggregationResultType, resultType)) + } + /** * Applies the given window function to each window. The window function is called for each * evaluation of the window. The output of the window function is @@ -355,6 +442,36 @@ class AllWindowedStream[T, W <: Window](javaStream: JavaAllWStream[T, W]) { implicitly[TypeInformation[R]])) } + /** + * Applies the given window function to each window. The window function is called for each + * evaluation of the window for each key individually. The output of the window function is + * interpreted as a regular non-windowed stream. + * + * Arriving data is pre-aggregated using the given pre-aggregation folder. + * + * @param initialValue Initial value of the fold + * @param preAggregator The reduce function that is used for pre-aggregation + * @param windowFunction The process window function. + * @return The data stream that is the result of applying the window function to the window. + */ + def fold[ACC: TypeInformation, R: TypeInformation]( + initialValue: ACC, + preAggregator: FoldFunction[T, ACC], + windowFunction: ProcessAllWindowFunction[ACC, R, W]): DataStream[R] = { + + val cleanFolder = clean(preAggregator) + val cleanWindowFunction = clean(windowFunction) + + val applyFunction = new ScalaProcessAllWindowFunctionWrapper[ACC, R, W](cleanWindowFunction) + + asScalaStream(javaStream.fold( + initialValue, + cleanFolder, + applyFunction, + implicitly[TypeInformation[ACC]], + implicitly[TypeInformation[R]])) + } + /** * Applies the given window function to each window. The window function is called for each * evaluation of the window for each key individually. The output of the window function is @@ -390,8 +507,64 @@ class AllWindowedStream[T, W <: Window](javaStream: JavaAllWStream[T, W]) { asScalaStream(javaStream.fold(initialValue, folder, applyFunction, accType, returnType)) } + /** + * Applies the given window function to each window. The window function is called for each + * evaluation of the window for each key individually. The output of the window function is + * interpreted as a regular non-windowed stream. + * + * Arriving data is pre-aggregated using the given pre-aggregation folder. + * + * @param initialValue Initial value of the fold + * @param preAggregator The reduce function that is used for pre-aggregation + * @param windowFunction The window function. + * @return The data stream that is the result of applying the window function to the window. + */ + def fold[ACC: TypeInformation, R: TypeInformation]( + initialValue: ACC, + preAggregator: (ACC, T) => ACC, + windowFunction: ProcessAllWindowFunction[ACC, R, W]): DataStream[R] = { + + if (preAggregator == null) { + throw new NullPointerException("Reduce function must not be null.") + } + if (windowFunction == null) { + throw new NullPointerException("WindowApply function must not be null.") + } + + val cleanFolder = clean(preAggregator) + val cleanWindowFunction = clean(windowFunction) + + val folder = new ScalaFoldFunction[T, ACC](cleanFolder) + val applyFunction = new ScalaProcessAllWindowFunctionWrapper[ACC, R, W](cleanWindowFunction) + + val accType: TypeInformation[ACC] = implicitly[TypeInformation[ACC]] + val returnType: TypeInformation[R] = implicitly[TypeInformation[R]] + asScalaStream(javaStream.fold(initialValue, folder, applyFunction, accType, returnType)) + } + // ---------------------------- apply() ------------------------------------- + /** + * Applies the given window function to each window. The window function is called for each + * evaluation of the window for each key individually. The output of the window function is + * interpreted as a regular non-windowed stream. + * + * Not that this function requires that all data in the windows is buffered until the window + * is evaluated, as the function provides no means of pre-aggregation. + * + * @param function The process window function. + * @return The data stream that is the result of applying the window function to the window. + */ + @PublicEvolving + def process[R: TypeInformation]( + function: ProcessAllWindowFunction[T, R, W]): DataStream[R] = { + + val cleanedFunction = clean(function) + val javaFunction = new ScalaProcessAllWindowFunctionWrapper[T, R, W](cleanedFunction) + + asScalaStream(javaStream.process(javaFunction, implicitly[TypeInformation[R]])) + } + /** * Applies the given window function to each window. The window function is called for each * evaluation of the window for each key individually. The output of the window function is diff --git a/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/function/ProcessAllWindowFunction.scala b/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/function/ProcessAllWindowFunction.scala new file mode 100644 index 00000000000000..163117be9cb68f --- /dev/null +++ b/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/function/ProcessAllWindowFunction.scala @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.streaming.api.scala.function + +import java.io.Serializable + +import org.apache.flink.annotation.PublicEvolving +import org.apache.flink.api.common.functions.Function +import org.apache.flink.streaming.api.windowing.windows.Window +import org.apache.flink.util.Collector + +/** + * Base abstract class for functions that are evaluated over keyed (grouped) + * windows using a context for retrieving extra information. + * + * @tparam IN The type of the input value. + * @tparam OUT The type of the output value. + * @tparam W The type of the window. + */ +@PublicEvolving +abstract class ProcessAllWindowFunction[IN, OUT, W <: Window] extends Function with Serializable { + /** + * Evaluates the window and outputs none or several elements. + * + * @param context The context in which the window is being evaluated. + * @param elements The elements in the window being evaluated. + * @param out A collector for emitting elements. + * @throws Exception The function may throw exceptions to fail the program and trigger recovery. + */ + @throws[Exception] + def process(context: Context, elements: Iterable[IN], out: Collector[OUT]) + + /** + * The context holding window metadata + */ + abstract class Context { + /** + * @return The window that is being evaluated. + */ + def window: W + } + +} diff --git a/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/function/RichProcessAllWindowFunction.scala b/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/function/RichProcessAllWindowFunction.scala new file mode 100644 index 00000000000000..22d64a83698d3c --- /dev/null +++ b/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/function/RichProcessAllWindowFunction.scala @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.streaming.api.scala.function + +import java.beans.Transient + +import org.apache.flink.annotation.Public +import org.apache.flink.api.common.functions.{IterationRuntimeContext, RichFunction, RuntimeContext} +import org.apache.flink.configuration.Configuration +import org.apache.flink.streaming.api.windowing.windows.Window + +/** + * Base abstract class for functions that are evaluated over + * keyed (grouped) windows using a context for retrieving extra information. + * + * @tparam IN The type of the input value. + * @tparam OUT The type of the output value. + * @tparam W The type of the window. + */ +@Public +abstract class RichProcessAllWindowFunction[IN, OUT, W <: Window] + extends ProcessAllWindowFunction[IN, OUT, W] + with RichFunction { + + @Transient + private var runtimeContext: RuntimeContext = null + + // -------------------------------------------------------------------------------------------- + // Runtime context access + // -------------------------------------------------------------------------------------------- + + override def setRuntimeContext(t: RuntimeContext) { + this.runtimeContext = t + } + + override def getRuntimeContext: RuntimeContext = { + if (this.runtimeContext != null) { + this.runtimeContext + } + else { + throw new IllegalStateException("The runtime context has not been initialized.") + } + } + + override def getIterationRuntimeContext: IterationRuntimeContext = { + if (this.runtimeContext == null) { + throw new IllegalStateException("The runtime context has not been initialized.") + } + else { + this.runtimeContext match { + case iterationRuntimeContext: IterationRuntimeContext => iterationRuntimeContext + case _ => + throw new IllegalStateException("This stub is not part of an iteration step function.") + } + } + } + + // -------------------------------------------------------------------------------------------- + // Default life cycle methods + // -------------------------------------------------------------------------------------------- + + @throws[Exception] + override def open(parameters: Configuration) { + } + + @throws[Exception] + override def close() { + } +} + diff --git a/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/function/util/ScalaProcessWindowFunctionWrapper.scala b/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/function/util/ScalaProcessWindowFunctionWrapper.scala index 23293a6f0c9911..a4fec64a06127b 100644 --- a/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/function/util/ScalaProcessWindowFunctionWrapper.scala +++ b/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/function/util/ScalaProcessWindowFunctionWrapper.scala @@ -18,8 +18,16 @@ package org.apache.flink.streaming.api.scala.function.util +import org.apache.flink.api.common.functions.RuntimeContext +import org.apache.flink.configuration.Configuration import org.apache.flink.streaming.api.functions.windowing.{ProcessWindowFunction => JProcessWindowFunction} -import org.apache.flink.streaming.api.scala.function.ProcessWindowFunction +import org.apache.flink.streaming.api.functions.windowing.{RichProcessWindowFunction => JRichProcessWindowFunction} +import org.apache.flink.streaming.api.functions.windowing.{RichProcessAllWindowFunction => JRichProcessAllWindowFunction} +import org.apache.flink.streaming.api.functions.windowing.{ProcessAllWindowFunction => JProcessAllWindowFunction} +import org.apache.flink.streaming.api.scala.function.{ProcessWindowFunction => ScalaProcessWindowFunction} +import org.apache.flink.streaming.api.scala.function.{ProcessAllWindowFunction => ScalaProcessAllWindowFunction} +import org.apache.flink.streaming.api.scala.function.{RichProcessWindowFunction => ScalaRichProcessWindowFunction} +import org.apache.flink.streaming.api.scala.function.{RichProcessAllWindowFunction => ScalaRichProcessAllWindowFunction} import org.apache.flink.streaming.api.windowing.windows.Window import org.apache.flink.util.Collector @@ -34,8 +42,8 @@ import scala.collection.JavaConverters._ * - Java WindowFunction: java.lang.Iterable */ final class ScalaProcessWindowFunctionWrapper[IN, OUT, KEY, W <: Window]( - private[this] val func: ProcessWindowFunction[IN, OUT, KEY, W]) - extends JProcessWindowFunction[IN, OUT, KEY, W] { + private[this] val func: ScalaProcessWindowFunction[IN, OUT, KEY, W]) + extends JRichProcessWindowFunction[IN, OUT, KEY, W] { override def process( key: KEY, @@ -47,4 +55,75 @@ final class ScalaProcessWindowFunctionWrapper[IN, OUT, KEY, W <: Window]( } func.process(key, ctx, elements.asScala, out) } + + override def setRuntimeContext(t: RuntimeContext): Unit = { + super.setRuntimeContext(t) + func match { + case rfunc: ScalaRichProcessWindowFunction[IN, OUT, KEY, W] => rfunc.setRuntimeContext(t) + case _ => + } + } + + override def open(parameters: Configuration): Unit = { + super.open(parameters) + func match { + case rfunc: ScalaRichProcessWindowFunction[IN, OUT, KEY, W] => rfunc.open(parameters) + case _ => + } + } + + override def close(): Unit = { + super.close() + func match { + case rfunc: ScalaRichProcessWindowFunction[IN, OUT, KEY, W] => rfunc.close() + case _ => + } + } +} + +/** + * A wrapper function that exposes a Scala ProcessWindowFunction + * as a ProcessWindowFunction function. + * + * The Scala and Java Window functions differ in their type of "Iterable": + * - Scala WindowFunction: scala.Iterable + * - Java WindowFunction: java.lang.Iterable + */ +final class ScalaProcessAllWindowFunctionWrapper[IN, OUT, W <: Window]( + private[this] val func: ScalaProcessAllWindowFunction[IN, OUT, W]) + extends JRichProcessAllWindowFunction[IN, OUT, W] { + + override def process( + context: JProcessAllWindowFunction[IN, OUT, W]#Context, + elements: java.lang.Iterable[IN], + out: Collector[OUT]): Unit = { + val ctx = new func.Context { + override def window = context.window + } + func.process(ctx, elements.asScala, out) + } + + override def setRuntimeContext(t: RuntimeContext): Unit = { + super.setRuntimeContext(t) + func match { + case rfunc : ScalaRichProcessAllWindowFunction[IN, OUT, W] => rfunc.setRuntimeContext(t) + case _ => + } + } + + override def open(parameters: Configuration): Unit = { + super.open(parameters) + func match { + case rfunc : ScalaRichProcessAllWindowFunction[IN, OUT, W] => rfunc.open(parameters) + case _ => + } + } + + override def close(): Unit = { + super.close() + func match { + case rfunc : ScalaRichProcessAllWindowFunction[IN, OUT, W] => rfunc.close() + case _ => + } + } } diff --git a/flink-streaming-scala/src/test/scala/org/apache/flink/streaming/api/scala/AllWindowTranslationTest.scala b/flink-streaming-scala/src/test/scala/org/apache/flink/streaming/api/scala/AllWindowTranslationTest.scala index 7e067a025349b7..ee9f50c6b1633a 100644 --- a/flink-streaming-scala/src/test/scala/org/apache/flink/streaming/api/scala/AllWindowTranslationTest.scala +++ b/flink-streaming-scala/src/test/scala/org/apache/flink/streaming/api/scala/AllWindowTranslationTest.scala @@ -26,7 +26,7 @@ import org.apache.flink.api.common.typeinfo.{BasicTypeInfo, TypeInformation} import org.apache.flink.api.java.functions.KeySelector import org.apache.flink.streaming.api.TimeCharacteristic import org.apache.flink.streaming.api.operators.OneInputStreamOperator -import org.apache.flink.streaming.api.scala.function.{WindowFunction, AllWindowFunction} +import org.apache.flink.streaming.api.scala.function.{AllWindowFunction, ProcessAllWindowFunction, WindowFunction} import org.apache.flink.streaming.api.transformations.OneInputTransformation import org.apache.flink.streaming.api.windowing.assigners._ import org.apache.flink.streaming.api.windowing.evictors.CountEvictor @@ -355,6 +355,85 @@ class AllWindowTranslationTest { ("hello", 1)) } + @Test + def testReduceWithProcessWindowFunctionEventTime() { + val env = StreamExecutionEnvironment.getExecutionEnvironment + env.setStreamTimeCharacteristic(TimeCharacteristic.IngestionTime) + + val source = env.fromElements(("hello", 1), ("hello", 2)) + + val window1 = source + .windowAll(TumblingEventTimeWindows.of(Time.seconds(1))) + .reduce( + new DummyReducer, new ProcessAllWindowFunction[(String, Int), (String, Int), TimeWindow] { + override def process(context: Context, + elements: Iterable[(String, Int)], + out: Collector[(String, Int)]): Unit = { + elements foreach ( x => out.collect(x)) + } + }) + + val transform = window1 + .javaStream + .getTransformation + .asInstanceOf[OneInputTransformation[(String, Int), (String, Int)]] + + val operator = transform.getOperator + assertTrue(operator.isInstanceOf[WindowOperator[_, _, _, _, _ <: Window]]) + + val winOperator = operator + .asInstanceOf[WindowOperator[String, (String, Int), _, (String, Int), _ <: Window]] + + assertTrue(winOperator.getTrigger.isInstanceOf[EventTimeTrigger]) + assertTrue(winOperator.getWindowAssigner.isInstanceOf[TumblingEventTimeWindows]) + assertTrue(winOperator.getStateDescriptor.isInstanceOf[ReducingStateDescriptor[_]]) + + processElementAndEnsureOutput[String, (String, Int), (String, Int)]( + winOperator, + winOperator.getKeySelector, + BasicTypeInfo.STRING_TYPE_INFO, + ("hello", 1)) + } + + @Test + def testReduceWithProcessWindowFunctionProcessingTime() { + val env = StreamExecutionEnvironment.getExecutionEnvironment + env.setStreamTimeCharacteristic(TimeCharacteristic.ProcessingTime) + + val source = env.fromElements(("hello", 1), ("hello", 2)) + + val window1 = source + .windowAll(TumblingProcessingTimeWindows.of(Time.seconds(1))) + .reduce( + new DummyReducer, new ProcessAllWindowFunction[(String, Int), (String, Int), TimeWindow] { + override def process( + context: Context, + input: Iterable[(String, Int)], + out: Collector[(String, Int)]): Unit = input foreach ( x => out.collect(x)) + }) + + val transform = window1 + .javaStream + .getTransformation + .asInstanceOf[OneInputTransformation[(String, Int), (String, Int)]] + + val operator = transform.getOperator + assertTrue(operator.isInstanceOf[WindowOperator[_, _, _, _, _ <: Window]]) + + val winOperator = operator + .asInstanceOf[WindowOperator[String, (String, Int), _, (String, Int), _ <: Window]] + + assertTrue(winOperator.getTrigger.isInstanceOf[ProcessingTimeTrigger]) + assertTrue(winOperator.getWindowAssigner.isInstanceOf[TumblingProcessingTimeWindows]) + assertTrue(winOperator.getStateDescriptor.isInstanceOf[ReducingStateDescriptor[_]]) + + processElementAndEnsureOutput[String, (String, Int), (String, Int)]( + winOperator, + winOperator.getKeySelector, + BasicTypeInfo.STRING_TYPE_INFO, + ("hello", 1)) + } + @Test def testApplyWithPreReducerEventTime() { val env = StreamExecutionEnvironment.getExecutionEnvironment @@ -565,6 +644,72 @@ class AllWindowTranslationTest { ("hello", 1)) } + @Test + def testAggregateWithProcessWindowFunctionEventTime() { + val env = StreamExecutionEnvironment.getExecutionEnvironment + env.setStreamTimeCharacteristic(TimeCharacteristic.IngestionTime) + + val source = env.fromElements(("hello", 1), ("hello", 2)) + + val window1 = source + .windowAll(TumblingEventTimeWindows.of(Time.seconds(1))) + .aggregate(new DummyAggregator(), new TestProcessAllWindowFunction()) + + val transform = window1 + .javaStream + .getTransformation + .asInstanceOf[OneInputTransformation[(String, Int), (String, Int)]] + + val operator = transform.getOperator + assertTrue(operator.isInstanceOf[WindowOperator[_, _, _, _, _ <: Window]]) + + val winOperator = operator + .asInstanceOf[WindowOperator[String, (String, Int), _, (String, Int), _ <: Window]] + + assertTrue(winOperator.getTrigger.isInstanceOf[EventTimeTrigger]) + assertTrue(winOperator.getWindowAssigner.isInstanceOf[TumblingEventTimeWindows]) + assertTrue(winOperator.getStateDescriptor.isInstanceOf[AggregatingStateDescriptor[_, _, _]]) + + processElementAndEnsureOutput[String, (String, Int), (String, Int)]( + winOperator, + winOperator.getKeySelector, + BasicTypeInfo.STRING_TYPE_INFO, + ("hello", 1)) + } + + @Test + def testAggregateWithProcessWindowFunctionProcessingTime() { + val env = StreamExecutionEnvironment.getExecutionEnvironment + env.setStreamTimeCharacteristic(TimeCharacteristic.ProcessingTime) + + val source = env.fromElements(("hello", 1), ("hello", 2)) + + val window1 = source + .windowAll(TumblingProcessingTimeWindows.of(Time.seconds(1))) + .aggregate(new DummyAggregator(), new TestProcessAllWindowFunction()) + + val transform = window1 + .javaStream + .getTransformation + .asInstanceOf[OneInputTransformation[(String, Int), (String, Int)]] + + val operator = transform.getOperator + assertTrue(operator.isInstanceOf[WindowOperator[_, _, _, _, _ <: Window]]) + + val winOperator = operator + .asInstanceOf[WindowOperator[String, (String, Int), _, (String, Int), _ <: Window]] + + assertTrue(winOperator.getTrigger.isInstanceOf[ProcessingTimeTrigger]) + assertTrue(winOperator.getWindowAssigner.isInstanceOf[TumblingProcessingTimeWindows]) + assertTrue(winOperator.getStateDescriptor.isInstanceOf[AggregatingStateDescriptor[_, _, _]]) + + processElementAndEnsureOutput[String, (String, Int), (String, Int)]( + winOperator, + winOperator.getKeySelector, + BasicTypeInfo.STRING_TYPE_INFO, + ("hello", 1)) + } + @Test def testAggregateWithWindowFunctionEventTimeWithScalaFunction() { val env = StreamExecutionEnvironment.getExecutionEnvironment @@ -788,6 +933,88 @@ class AllWindowTranslationTest { ("hello", 1)) } + @Test + def testFoldWithProcessWindowFunctionEventTime() { + val env = StreamExecutionEnvironment.getExecutionEnvironment + env.setStreamTimeCharacteristic(TimeCharacteristic.IngestionTime) + + val source = env.fromElements(("hello", 1), ("hello", 2)) + + val window1 = source + .windowAll(TumblingEventTimeWindows.of(Time.seconds(1), Time.milliseconds(100))) + .fold( + ("", "", 1), + new DummyFolder, + new ProcessAllWindowFunction[(String, String, Int), (String, Int), TimeWindow] { + override def process( + context: Context, + input: Iterable[(String, String, Int)], + out: Collector[(String, Int)]): Unit = input foreach {x => out.collect((x._1, x._3))} + }) + + val transform = window1 + .javaStream + .getTransformation + .asInstanceOf[OneInputTransformation[(String, Int), (String, Int)]] + + val operator = transform.getOperator + assertTrue(operator.isInstanceOf[WindowOperator[_, _, _, _, _ <: Window]]) + + val winOperator = operator + .asInstanceOf[WindowOperator[String, (String, Int), _, (String, Int), _ <: Window]] + + assertTrue(winOperator.getTrigger.isInstanceOf[EventTimeTrigger]) + assertTrue(winOperator.getWindowAssigner.isInstanceOf[TumblingEventTimeWindows]) + assertTrue(winOperator.getStateDescriptor.isInstanceOf[FoldingStateDescriptor[_, _]]) + + processElementAndEnsureOutput[String, (String, Int), (String, Int)]( + winOperator, + winOperator.getKeySelector, + BasicTypeInfo.STRING_TYPE_INFO, + ("hello", 1)) + } + + @Test + def testFoldWithProcessWindowFunctionProcessingTime() { + val env = StreamExecutionEnvironment.getExecutionEnvironment + env.setStreamTimeCharacteristic(TimeCharacteristic.ProcessingTime) + + val source = env.fromElements(("hello", 1), ("hello", 2)) + + val window1 = source + .windowAll(TumblingProcessingTimeWindows.of(Time.seconds(1), Time.milliseconds(100))) + .fold( + ("", "", 1), + new DummyFolder, + new ProcessAllWindowFunction[(String, String, Int), (String, Int), TimeWindow] { + override def process( + context: Context, + input: Iterable[(String, String, Int)], + out: Collector[(String, Int)]): Unit = input foreach {x => out.collect((x._1, x._3))} + }) + + val transform = window1 + .javaStream + .getTransformation + .asInstanceOf[OneInputTransformation[(String, Int), (String, Int)]] + + val operator = transform.getOperator + assertTrue(operator.isInstanceOf[WindowOperator[_, _, _, _, _ <: Window]]) + + val winOperator = operator + .asInstanceOf[WindowOperator[String, (String, Int), _, (String, Int), _ <: Window]] + + assertTrue(winOperator.getTrigger.isInstanceOf[ProcessingTimeTrigger]) + assertTrue(winOperator.getWindowAssigner.isInstanceOf[TumblingProcessingTimeWindows]) + assertTrue(winOperator.getStateDescriptor.isInstanceOf[FoldingStateDescriptor[_, _]]) + + processElementAndEnsureOutput[String, (String, Int), (String, Int)]( + winOperator, + winOperator.getKeySelector, + BasicTypeInfo.STRING_TYPE_INFO, + ("hello", 1)) + } + @Test def testApplyWithPreFolderEventTime() { val env = StreamExecutionEnvironment.getExecutionEnvironment @@ -950,6 +1177,84 @@ class AllWindowTranslationTest { ("hello", 1)) } + @Test + def testProcessEventTime() { + val env = StreamExecutionEnvironment.getExecutionEnvironment + env.setStreamTimeCharacteristic(TimeCharacteristic.IngestionTime) + + val source = env.fromElements(("hello", 1), ("hello", 2)) + + val window1 = source + .windowAll(TumblingEventTimeWindows.of(Time.seconds(1), Time.milliseconds(100))) + .process( + new ProcessAllWindowFunction[(String, Int), (String, Int), TimeWindow] { + override def process( + context: Context, + input: Iterable[(String, Int)], + out: Collector[(String, Int)]): Unit = input foreach {x => out.collect((x._1, x._2))} + }) + + val transform = window1 + .javaStream + .getTransformation + .asInstanceOf[OneInputTransformation[(String, Int), (String, Int)]] + + val operator = transform.getOperator + assertTrue(operator.isInstanceOf[WindowOperator[_, _, _, _, _ <: Window]]) + + val winOperator = operator + .asInstanceOf[WindowOperator[String, (String, Int), _, (String, Int), _ <: Window]] + + assertTrue(winOperator.getTrigger.isInstanceOf[EventTimeTrigger]) + assertTrue(winOperator.getWindowAssigner.isInstanceOf[TumblingEventTimeWindows]) + assertTrue(winOperator.getStateDescriptor.isInstanceOf[ListStateDescriptor[_]]) + + processElementAndEnsureOutput[String, (String, Int), (String, Int)]( + winOperator, + winOperator.getKeySelector, + BasicTypeInfo.STRING_TYPE_INFO, + ("hello", 1)) + } + + @Test + def testProcessProcessingTimeTime() { + val env = StreamExecutionEnvironment.getExecutionEnvironment + env.setStreamTimeCharacteristic(TimeCharacteristic.ProcessingTime) + + val source = env.fromElements(("hello", 1), ("hello", 2)) + + val window1 = source + .windowAll(TumblingProcessingTimeWindows.of(Time.seconds(1), Time.milliseconds(100))) + .process( + new ProcessAllWindowFunction[(String, Int), (String, Int), TimeWindow] { + override def process( + context: Context, + input: Iterable[(String, Int)], + out: Collector[(String, Int)]): Unit = input foreach {x => out.collect((x._1, x._2))} + }) + + val transform = window1 + .javaStream + .getTransformation + .asInstanceOf[OneInputTransformation[(String, Int), (String, Int)]] + + val operator = transform.getOperator + assertTrue(operator.isInstanceOf[WindowOperator[_, _, _, _, _ <: Window]]) + + val winOperator = operator + .asInstanceOf[WindowOperator[String, (String, Int), _, (String, Int), _ <: Window]] + + assertTrue(winOperator.getTrigger.isInstanceOf[ProcessingTimeTrigger]) + assertTrue(winOperator.getWindowAssigner.isInstanceOf[TumblingProcessingTimeWindows]) + assertTrue(winOperator.getStateDescriptor.isInstanceOf[ListStateDescriptor[_]]) + + processElementAndEnsureOutput[String, (String, Int), (String, Int)]( + winOperator, + winOperator.getKeySelector, + BasicTypeInfo.STRING_TYPE_INFO, + ("hello", 1)) + } + @Test def testApplyEventTimeWithScalaFunction() { val env = StreamExecutionEnvironment.getExecutionEnvironment @@ -1094,6 +1399,46 @@ class AllWindowTranslationTest { ("hello", 1)) } + @Test + def testProcessWithCustomTrigger() { + val env = StreamExecutionEnvironment.getExecutionEnvironment + env.setStreamTimeCharacteristic(TimeCharacteristic.IngestionTime) + + val source = env.fromElements(("hello", 1), ("hello", 2)) + + val window1 = source + .windowAll(TumblingEventTimeWindows.of(Time.seconds(1), Time.milliseconds(100))) + .trigger(CountTrigger.of(1)) + .process( + new ProcessAllWindowFunction[(String, Int), (String, Int), TimeWindow] { + override def process( + context: Context, + input: Iterable[(String, Int)], + out: Collector[(String, Int)]): Unit = input foreach {x => out.collect((x._1, x._2))} + }) + + val transform = window1 + .javaStream + .getTransformation + .asInstanceOf[OneInputTransformation[(String, Int), (String, Int)]] + + val operator = transform.getOperator + assertTrue(operator.isInstanceOf[WindowOperator[_, _, _, _, _ <: Window]]) + + val winOperator = operator + .asInstanceOf[WindowOperator[String, (String, Int), _, (String, Int), _ <: Window]] + + assertTrue(winOperator.getTrigger.isInstanceOf[CountTrigger[_]]) + assertTrue(winOperator.getWindowAssigner.isInstanceOf[TumblingEventTimeWindows]) + assertTrue(winOperator.getStateDescriptor.isInstanceOf[ListStateDescriptor[_]]) + + processElementAndEnsureOutput[String, (String, Int), (String, Int)]( + winOperator, + winOperator.getKeySelector, + BasicTypeInfo.STRING_TYPE_INFO, + ("hello", 1)) + } + @Test def testReduceWithEvictor() { val env = StreamExecutionEnvironment.getExecutionEnvironment @@ -1211,6 +1556,46 @@ class AllWindowTranslationTest { ("hello", 1)) } + @Test + def testProcessWithEvictor() { + val env = StreamExecutionEnvironment.getExecutionEnvironment + env.setStreamTimeCharacteristic(TimeCharacteristic.IngestionTime) + + val source = env.fromElements(("hello", 1), ("hello", 2)) + + val window1 = source + .windowAll(TumblingEventTimeWindows.of(Time.seconds(1), Time.milliseconds(100))) + .evictor(CountEvictor.of(100)) + .process( + new ProcessAllWindowFunction[(String, Int), (String, Int), TimeWindow] { + override def process( + context: Context, + input: Iterable[(String, Int)], + out: Collector[(String, Int)]): Unit = input foreach {x => out.collect((x._1, x._2))} + }) + + val transform = window1 + .javaStream + .getTransformation + .asInstanceOf[OneInputTransformation[(String, Int), (String, Int)]] + + val operator = transform.getOperator + assertTrue(operator.isInstanceOf[EvictingWindowOperator[_, _, _, _ <: Window]]) + + val winOperator = operator + .asInstanceOf[EvictingWindowOperator[String, (String, Int), (String, Int), _ <: Window]] + + assertTrue(winOperator.getTrigger.isInstanceOf[EventTimeTrigger]) + assertTrue(winOperator.getEvictor.isInstanceOf[CountEvictor[_]]) + assertTrue(winOperator.getWindowAssigner.isInstanceOf[TumblingEventTimeWindows]) + assertTrue(winOperator.getStateDescriptor.isInstanceOf[ListStateDescriptor[_]]) + + processElementAndEnsureOutput[String, (String, Int), (String, Int)]( + winOperator, + winOperator.getKeySelector, + BasicTypeInfo.STRING_TYPE_INFO, + ("hello", 1)) + } /** * Ensure that we get some output from the given operator when pushing in an element and @@ -1218,10 +1603,10 @@ class AllWindowTranslationTest { */ @throws[Exception] private def processElementAndEnsureOutput[K, IN, OUT]( - operator: OneInputStreamOperator[IN, OUT], - keySelector: KeySelector[IN, K], - keyType: TypeInformation[K], - element: IN) { + operator: OneInputStreamOperator[IN, OUT], + keySelector: KeySelector[IN, K], + keyType: TypeInformation[K], + element: IN) { val testHarness = new KeyedOneInputStreamOperatorTestHarness[K, IN, OUT](operator, keySelector, keyType) @@ -1243,7 +1628,8 @@ class AllWindowTranslationTest { } } -class TestAllWindowFunction extends AllWindowFunction[(String, Int), (String, Int), TimeWindow] { +class TestAllWindowFunction + extends AllWindowFunction[(String, Int), (String, Int), TimeWindow] { override def apply( window: TimeWindow, @@ -1253,3 +1639,15 @@ class TestAllWindowFunction extends AllWindowFunction[(String, Int), (String, In input.foreach(out.collect) } } + +class TestProcessAllWindowFunction + extends ProcessAllWindowFunction[(String, Int), (String, Int), TimeWindow] { + + override def process( + context: Context, + input: Iterable[(String, Int)], + out: Collector[(String, Int)]): Unit = { + + input.foreach(out.collect) + } +} diff --git a/flink-streaming-scala/src/test/scala/org/apache/flink/streaming/api/scala/WindowFoldITCase.scala b/flink-streaming-scala/src/test/scala/org/apache/flink/streaming/api/scala/WindowFoldITCase.scala index a23145c563e44c..dc387589da37bf 100644 --- a/flink-streaming-scala/src/test/scala/org/apache/flink/streaming/api/scala/WindowFoldITCase.scala +++ b/flink-streaming-scala/src/test/scala/org/apache/flink/streaming/api/scala/WindowFoldITCase.scala @@ -26,7 +26,7 @@ import org.apache.flink.streaming.api.TimeCharacteristic import org.apache.flink.streaming.api.functions.AssignerWithPunctuatedWatermarks import org.apache.flink.streaming.api.functions.sink.SinkFunction import org.apache.flink.streaming.api.functions.source.SourceFunction -import org.apache.flink.streaming.api.scala.testutils.{CheckingIdentityRichAllWindowFunction, CheckingIdentityRichProcessWindowFunction, CheckingIdentityRichWindowFunction} +import org.apache.flink.streaming.api.scala.testutils.{CheckingIdentityRichAllWindowFunction, CheckingIdentityRichProcessAllWindowFunction, CheckingIdentityRichProcessWindowFunction, CheckingIdentityRichWindowFunction} import org.apache.flink.streaming.api.watermark.Watermark import org.apache.flink.streaming.api.windowing.assigners.TumblingEventTimeWindows import org.apache.flink.streaming.api.windowing.time.Time @@ -150,7 +150,6 @@ class WindowFoldITCase extends StreamingMultipleProgramsTestBase { } @Test - @Ignore def testFoldWithProcessWindowFunction(): Unit = { WindowFoldITCase.testResults = mutable.MutableList() CheckingIdentityRichProcessWindowFunction.reset() @@ -310,6 +309,63 @@ class WindowFoldITCase extends StreamingMultipleProgramsTestBase { CheckingIdentityRichAllWindowFunction.checkRichMethodCalls() } + + @Test + def testFoldAllWithProcessWindowFunction(): Unit = { + WindowFoldITCase.testResults = mutable.MutableList() + CheckingIdentityRichProcessAllWindowFunction.reset() + + val foldFunc = new FoldFunction[(String, Int), (String, Int)] { + override def fold(accumulator: (String, Int), value: (String, Int)): (String, Int) = { + (accumulator._1 + value._1, accumulator._2 + value._2) + } + } + + val env = StreamExecutionEnvironment.getExecutionEnvironment + env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime) + env.setParallelism(1) + + val source1 = env.addSource(new SourceFunction[(String, Int)]() { + def run(ctx: SourceFunction.SourceContext[(String, Int)]) { + ctx.collect(("a", 0)) + ctx.collect(("a", 1)) + ctx.collect(("a", 2)) + ctx.collect(("b", 3)) + ctx.collect(("a", 3)) + ctx.collect(("b", 4)) + ctx.collect(("a", 4)) + ctx.collect(("b", 5)) + ctx.collect(("a", 5)) + + // source is finite, so it will have an implicit MAX watermark when it finishes + } + + def cancel() { + } + }).assignTimestampsAndWatermarks(new WindowFoldITCase.Tuple2TimestampExtractor) + + source1 + .windowAll(TumblingEventTimeWindows.of(Time.of(3, TimeUnit.MILLISECONDS))) + .fold( + ("R:", 0), + foldFunc, + new CheckingIdentityRichProcessAllWindowFunction[(String, Int), TimeWindow]()) + .addSink(new SinkFunction[(String, Int)]() { + def invoke(value: (String, Int)) { + WindowFoldITCase.testResults += value.toString + } + }) + + env.execute("Fold All-Window Test") + + val expectedResult = mutable.MutableList( + "(R:aaa,3)", + "(R:bababa,24)") + + assertEquals(expectedResult.sorted, WindowFoldITCase.testResults.sorted) + + CheckingIdentityRichProcessAllWindowFunction.checkRichMethodCalls() + } } diff --git a/flink-streaming-scala/src/test/scala/org/apache/flink/streaming/api/scala/WindowFunctionITCase.scala b/flink-streaming-scala/src/test/scala/org/apache/flink/streaming/api/scala/WindowFunctionITCase.scala index bfbe6ee1d64dfa..eb9f36131e059e 100644 --- a/flink-streaming-scala/src/test/scala/org/apache/flink/streaming/api/scala/WindowFunctionITCase.scala +++ b/flink-streaming-scala/src/test/scala/org/apache/flink/streaming/api/scala/WindowFunctionITCase.scala @@ -25,7 +25,7 @@ import org.apache.flink.streaming.api.TimeCharacteristic import org.apache.flink.streaming.api.functions.AssignerWithPunctuatedWatermarks import org.apache.flink.streaming.api.functions.sink.SinkFunction import org.apache.flink.streaming.api.functions.source.SourceFunction -import org.apache.flink.streaming.api.scala.testutils.{CheckingIdentityRichAllWindowFunction, CheckingIdentityRichProcessWindowFunction, CheckingIdentityRichWindowFunction} +import org.apache.flink.streaming.api.scala.testutils.{CheckingIdentityRichAllWindowFunction, CheckingIdentityRichProcessAllWindowFunction, CheckingIdentityRichProcessWindowFunction, CheckingIdentityRichWindowFunction} import org.apache.flink.streaming.api.watermark.Watermark import org.apache.flink.streaming.api.windowing.assigners.TumblingEventTimeWindows import org.apache.flink.streaming.api.windowing.time.Time @@ -87,7 +87,6 @@ class WindowFunctionITCase { } @Test - @Ignore def testRichProcessWindowFunction(): Unit = { WindowFunctionITCase.testResults = mutable.MutableList() CheckingIdentityRichProcessWindowFunction.reset() @@ -183,6 +182,54 @@ class WindowFunctionITCase { CheckingIdentityRichAllWindowFunction.checkRichMethodCalls() } + + @Test + def testRichProcessAllWindowFunction(): Unit = { + WindowFunctionITCase.testResults = mutable.MutableList() + CheckingIdentityRichProcessAllWindowFunction.reset() + + val env = StreamExecutionEnvironment.getExecutionEnvironment + env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime) + env.setParallelism(1) + + val source1 = env.addSource(new SourceFunction[(String, Int)]() { + def run(ctx: SourceFunction.SourceContext[(String, Int)]) { + ctx.collect(("a", 0)) + ctx.collect(("a", 1)) + ctx.collect(("a", 2)) + ctx.collect(("b", 3)) + ctx.collect(("b", 4)) + ctx.collect(("b", 5)) + ctx.collect(("a", 6)) + ctx.collect(("a", 7)) + ctx.collect(("a", 8)) + + // source is finite, so it will have an implicit MAX watermark when it finishes + } + + def cancel() {} + + }).assignTimestampsAndWatermarks(new WindowFunctionITCase.Tuple2TimestampExtractor) + + source1 + .windowAll(TumblingEventTimeWindows.of(Time.of(3, TimeUnit.MILLISECONDS))) + .process(new CheckingIdentityRichProcessAllWindowFunction[(String, Int), TimeWindow]()) + .addSink(new SinkFunction[(String, Int)]() { + def invoke(value: (String, Int)) { + WindowFunctionITCase.testResults += value.toString + } + }) + + env.execute("RichAllWindowFunction Test") + + val expectedResult = mutable.MutableList( + "(a,0)", "(a,1)", "(a,2)", "(a,6)", "(a,7)", "(a,8)", + "(b,3)", "(b,4)", "(b,5)") + + assertEquals(expectedResult.sorted, WindowFunctionITCase.testResults.sorted) + + CheckingIdentityRichProcessAllWindowFunction.checkRichMethodCalls() + } } object WindowFunctionITCase { diff --git a/flink-streaming-scala/src/test/scala/org/apache/flink/streaming/api/scala/WindowReduceITCase.scala b/flink-streaming-scala/src/test/scala/org/apache/flink/streaming/api/scala/WindowReduceITCase.scala index 5418108e1921ad..ee1dbfd608c5ac 100644 --- a/flink-streaming-scala/src/test/scala/org/apache/flink/streaming/api/scala/WindowReduceITCase.scala +++ b/flink-streaming-scala/src/test/scala/org/apache/flink/streaming/api/scala/WindowReduceITCase.scala @@ -26,7 +26,7 @@ import org.apache.flink.streaming.api.TimeCharacteristic import org.apache.flink.streaming.api.functions.AssignerWithPunctuatedWatermarks import org.apache.flink.streaming.api.functions.sink.SinkFunction import org.apache.flink.streaming.api.functions.source.SourceFunction -import org.apache.flink.streaming.api.scala.testutils.{CheckingIdentityRichAllWindowFunction, CheckingIdentityRichProcessWindowFunction, CheckingIdentityRichWindowFunction} +import org.apache.flink.streaming.api.scala.testutils.{CheckingIdentityRichAllWindowFunction, CheckingIdentityRichProcessAllWindowFunction, CheckingIdentityRichProcessWindowFunction, CheckingIdentityRichWindowFunction} import org.apache.flink.streaming.api.watermark.Watermark import org.apache.flink.streaming.api.windowing.assigners.TumblingEventTimeWindows import org.apache.flink.streaming.api.windowing.time.Time @@ -149,7 +149,6 @@ class WindowReduceITCase extends StreamingMultipleProgramsTestBase { } @Test - @Ignore def testReduceWithProcessWindowFunction(): Unit = { WindowReduceITCase.testResults = mutable.MutableList() CheckingIdentityRichProcessWindowFunction.reset() @@ -307,6 +306,62 @@ class WindowReduceITCase extends StreamingMultipleProgramsTestBase { CheckingIdentityRichAllWindowFunction.checkRichMethodCalls() } + + @Test + def testReduceAllWithProcessWindowFunction(): Unit = { + WindowReduceITCase.testResults = mutable.MutableList() + CheckingIdentityRichProcessAllWindowFunction.reset() + + val reduceFunc = new ReduceFunction[(String, Int)] { + override def reduce(a: (String, Int), b: (String, Int)): (String, Int) = { + (a._1 + b._1, a._2 + b._2) + } + } + + val env = StreamExecutionEnvironment.getExecutionEnvironment + env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime) + env.setParallelism(1) + + val source1 = env.addSource(new SourceFunction[(String, Int)]() { + def run(ctx: SourceFunction.SourceContext[(String, Int)]) { + ctx.collect(("a", 0)) + ctx.collect(("a", 1)) + ctx.collect(("a", 2)) + ctx.collect(("b", 3)) + ctx.collect(("a", 3)) + ctx.collect(("b", 4)) + ctx.collect(("a", 4)) + ctx.collect(("b", 5)) + ctx.collect(("a", 5)) + + // source is finite, so it will have an implicit MAX watermark when it finishes + } + + def cancel() { + } + }).assignTimestampsAndWatermarks(new WindowReduceITCase.Tuple2TimestampExtractor) + + source1 + .windowAll(TumblingEventTimeWindows.of(Time.of(3, TimeUnit.MILLISECONDS))) + .reduce( + reduceFunc, + new CheckingIdentityRichProcessAllWindowFunction[(String, Int), TimeWindow]()) + .addSink(new SinkFunction[(String, Int)]() { + def invoke(value: (String, Int)) { + WindowReduceITCase.testResults += value.toString + } + }) + + env.execute("Fold All-Window Test") + + val expectedResult = mutable.MutableList( + "(aaa,3)", + "(bababa,24)") + + assertEquals(expectedResult.sorted, WindowReduceITCase.testResults.sorted) + + CheckingIdentityRichProcessAllWindowFunction.checkRichMethodCalls() + } } object WindowReduceITCase { diff --git a/flink-streaming-scala/src/test/scala/org/apache/flink/streaming/api/scala/testutils/CheckingIdentityRichProcessAllWindowFunction.scala b/flink-streaming-scala/src/test/scala/org/apache/flink/streaming/api/scala/testutils/CheckingIdentityRichProcessAllWindowFunction.scala new file mode 100644 index 00000000000000..df005fa63a83a2 --- /dev/null +++ b/flink-streaming-scala/src/test/scala/org/apache/flink/streaming/api/scala/testutils/CheckingIdentityRichProcessAllWindowFunction.scala @@ -0,0 +1,81 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.streaming.api.scala.testutils + +import org.apache.flink.api.common.functions.RuntimeContext +import org.apache.flink.configuration.Configuration +import org.apache.flink.streaming.api.scala.function.RichProcessAllWindowFunction +import org.apache.flink.streaming.api.windowing.windows.Window +import org.apache.flink.util.Collector + + +class CheckingIdentityRichProcessAllWindowFunction[T, W <: Window] + extends RichProcessAllWindowFunction[T, T, W] { + + override def process(context: Context, input: Iterable[T], out: Collector[T]): Unit = { + for (value <- input) { + out.collect(value) + } + } + + override def open(conf: Configuration): Unit = { + super.open(conf) + CheckingIdentityRichProcessAllWindowFunction.openCalled = true + } + + override def close(): Unit = { + super.close() + CheckingIdentityRichProcessAllWindowFunction.closeCalled = true + } + + override def setRuntimeContext(context: RuntimeContext): Unit = { + super.setRuntimeContext(context) + CheckingIdentityRichProcessAllWindowFunction.contextSet = true + } +} + +object CheckingIdentityRichProcessAllWindowFunction { + + @volatile + private[CheckingIdentityRichProcessAllWindowFunction] var closeCalled = false + + @volatile + private[CheckingIdentityRichProcessAllWindowFunction] var openCalled = false + + @volatile + private[CheckingIdentityRichProcessAllWindowFunction] var contextSet = false + + def reset(): Unit = { + closeCalled = false + openCalled = false + contextSet = false + } + + def checkRichMethodCalls(): Unit = { + if (!contextSet) { + throw new AssertionError("context not set") + } + if (!openCalled) { + throw new AssertionError("open() not called") + } + if (!closeCalled) { + throw new AssertionError("close() not called") + } + } +} diff --git a/flink-tests/src/test/java/org/apache/flink/test/streaming/runtime/WindowFoldITCase.java b/flink-tests/src/test/java/org/apache/flink/test/streaming/runtime/WindowFoldITCase.java index 7d37d1aba07218..903179db28d563 100644 --- a/flink-tests/src/test/java/org/apache/flink/test/streaming/runtime/WindowFoldITCase.java +++ b/flink-tests/src/test/java/org/apache/flink/test/streaming/runtime/WindowFoldITCase.java @@ -27,6 +27,7 @@ import org.apache.flink.streaming.api.functions.AssignerWithPunctuatedWatermarks; import org.apache.flink.streaming.api.functions.sink.SinkFunction; import org.apache.flink.streaming.api.functions.source.SourceFunction; +import org.apache.flink.streaming.api.functions.windowing.ProcessAllWindowFunction; import org.apache.flink.streaming.api.functions.windowing.ProcessWindowFunction; import org.apache.flink.streaming.api.watermark.Watermark; import org.apache.flink.streaming.api.windowing.assigners.TumblingEventTimeWindows; @@ -253,6 +254,78 @@ public void invoke(Tuple2 value) throws Exception { Assert.assertEquals(expectedResult, testResults); } + @Test + public void testFoldProcessAllWindow() throws Exception { + + testResults = new ArrayList<>(); + + StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); + env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime); + env.setParallelism(1); + + DataStream> source1 = env.addSource(new SourceFunction>() { + private static final long serialVersionUID = 1L; + + @Override + public void run(SourceContext> ctx) throws Exception { + ctx.collect(Tuple2.of("a", 0)); + ctx.collect(Tuple2.of("a", 1)); + ctx.collect(Tuple2.of("a", 2)); + + ctx.collect(Tuple2.of("b", 3)); + ctx.collect(Tuple2.of("b", 4)); + ctx.collect(Tuple2.of("b", 5)); + + ctx.collect(Tuple2.of("a", 6)); + ctx.collect(Tuple2.of("a", 7)); + ctx.collect(Tuple2.of("a", 8)); + + // source is finite, so it will have an implicit MAX watermark when it finishes + } + + @Override + public void cancel() {} + + }).assignTimestampsAndWatermarks(new Tuple2TimestampExtractor()); + + source1 + .windowAll(TumblingEventTimeWindows.of(Time.of(3, TimeUnit.MILLISECONDS))) + .fold(Tuple2.of(0, "R:"), new FoldFunction, Tuple2>() { + @Override + public Tuple2 fold(Tuple2 accumulator, Tuple2 value) throws Exception { + accumulator.f1 += value.f0; + accumulator.f0 += value.f1; + return accumulator; + } + }, new ProcessAllWindowFunction, Tuple3, TimeWindow>() { + @Override + public void process(Context context, Iterable> elements, Collector> out) throws Exception { + int i = 0; + for (Tuple2 in : elements) { + out.collect(new Tuple3<>(in.f1, in.f0, i++)); + } + } + }) + .addSink(new SinkFunction>() { + @Override + public void invoke(Tuple3 value) throws Exception { + testResults.add(value.toString()); + } + }); + + env.execute("Fold Process Window Test"); + + List expectedResult = Arrays.asList( + "(R:aaa,3,0)", + "(R:aaa,21,0)", + "(R:bbb,12,0)"); + + Collections.sort(expectedResult); + Collections.sort(testResults); + + Assert.assertEquals(expectedResult, testResults); + } + private static class Tuple2TimestampExtractor implements AssignerWithPunctuatedWatermarks> { @Override