From c95e91e457f6cdc2df92a0c9a38c9a932be9f6a9 Mon Sep 17 00:00:00 2001 From: Aljoscha Krettek Date: Mon, 25 Jan 2016 12:33:51 +0100 Subject: [PATCH 1/3] [FLINK-3201] Enhance Partitioned State Interface with State Types Add new state types ValueState, ListState and ReducingState, where ListState and ReducingState derive from interface MergingState. ValueState behaves exactly the same as OperatorState. MergingState is a stateful list to which elements can be added and for which the elements that it contains can be obtained. If using a ListState the list of elements is actually kept, for a ReducingState a reduce function is used to combine all added elements into one. To create a ValueState the user passes a ValueStateIdentifier to StreamingRuntimeContext.getPartitionedState() while they would pass a ListStateIdentifier or ReducingStateIdentifier for the other state types. This change is necessary to give the system more information about the nature of the operator state. We want this to be able to do incremental snapshots. This would not be possible, for example, if the user had a List as a state. Inside OperatorState this list would be opaque and Flink could not create good incremental snapshots. This also refactors the StateBackend. Before, the logic for partitioned state was spread out over StreamingRuntimeContext, AbstractStreamOperator and StateBackend. Now it is consolidated in StateBackend. This also adds support for partitioned state in two-input operators. --- .../streaming/state/DbStateBackend.java | 77 ++- ...zyDbKvState.java => LazyDbValueState.java} | 153 ++++-- .../contrib/streaming/state/MySqlAdapter.java | 2 +- .../streaming/state/DbStateBackendTest.java | 344 +++++++++--- .../contrib/streaming/state/DerbyAdapter.java | 1 - .../src/test/resources/log4j-test.properties | 4 +- .../api/common/functions/RuntimeContext.java | 71 ++- .../util/AbstractRuntimeUDFContext.java | 17 +- .../flink/api/common/state/ListState.java | 33 ++ .../api/common/state/ListStateDescriptor.java | 87 +++ .../flink/api/common/state/MergingState.java | 66 +++ .../flink/api/common/state/OperatorState.java | 1 + .../flink/api/common/state/ReducingState.java | 35 ++ .../common/state/ReducingStateDescriptor.java | 106 ++++ .../apache/flink/api/common/state/State.java | 30 ++ .../flink/api/common/state/StateBackend.java | 50 ++ .../api/common/state/StateDescriptor.java | 66 +++ .../flink/api/common/state/ValueState.java | 69 +++ .../common/state/ValueStateDescriptor.java | 166 ++++++ .../base/TypeSerializerSingleton.java | 2 +- .../examples/windowing/SessionWindowing.java | 6 +- .../flink/hdfstests/FileStateBackendTest.java | 37 +- .../runtime/state/AbstractHeapKvState.java | 146 ------ .../runtime/state/AbstractHeapState.java | 164 ++++++ .../runtime/state/AbstractStateBackend.java | 406 ++++++++++++++ .../runtime/state/ArrayListSerializer.java | 125 +++++ .../runtime/state/CheckpointListener.java | 4 +- .../flink/runtime/state/GenericListState.java | 132 +++++ .../runtime/state/GenericReducingState.java | 129 +++++ .../apache/flink/runtime/state/KvState.java | 33 +- .../flink/runtime/state/KvStateSnapshot.java | 25 +- .../flink/runtime/state/StateBackend.java | 220 -------- .../runtime/state/StateBackendFactory.java | 4 +- ...tate.java => AbstractFileStateHandle.java} | 4 +- .../state/filesystem/AbstractFsState.java | 95 ++++ .../filesystem/AbstractFsStateSnapshot.java | 136 +++++ .../FileSerializableStateHandle.java | 2 +- .../filesystem/FileStreamStateHandle.java | 2 +- .../state/filesystem/FsHeapKvState.java | 86 --- .../filesystem/FsHeapKvStateSnapshot.java | 107 ---- .../runtime/state/filesystem/FsListState.java | 140 +++++ .../state/filesystem/FsReducingState.java | 149 ++++++ .../state/filesystem/FsStateBackend.java | 34 +- .../state/filesystem/FsValueState.java | 126 +++++ .../state/memory/AbstractMemState.java | 82 +++ .../memory/AbstractMemStateSnapshot.java | 127 +++++ .../runtime/state/memory/MemHeapKvState.java | 52 -- .../runtime/state/memory/MemListState.java | 111 ++++ .../state/memory/MemReducingState.java | 123 +++++ .../runtime/state/memory/MemValueState.java | 100 ++++ .../memory/MemoryHeapKvStateSnapshot.java | 107 ---- .../state/memory/MemoryStateBackend.java | 40 +- .../runtime/state/FileStateBackendTest.java | 280 ++-------- .../FsCheckpointStateOutputStreamTest.java | 4 +- .../runtime/state/MemoryStateBackendTest.java | 182 +------ .../runtime/state/StateBackendTestBase.java | 494 ++++++++++++++++++ .../streaming/connectors/fs/RollingSink.java | 4 +- .../kafka/FlinkKafkaConsumerBase.java | 4 +- .../kafka/KafkaConsumerTestBase.java | 4 +- .../testutils/FailingIdentityMapper.java | 4 +- .../kafka/testutils/MockRuntimeContext.java | 13 +- .../api/datastream/ConnectedStreams.java | 15 + .../StreamExecutionEnvironment.java | 13 +- .../MessageAcknowledgingSourceBase.java | 4 +- .../streaming/api/graph/StreamConfig.java | 14 +- .../streaming/api/graph/StreamGraph.java | 19 +- .../api/graph/StreamGraphGenerator.java | 13 +- .../flink/streaming/api/graph/StreamNode.java | 19 +- .../api/graph/StreamingJobGraphGenerator.java | 3 +- .../api/operators/AbstractStreamOperator.java | 208 +++----- .../operators/AbstractUdfStreamOperator.java | 11 +- .../api/operators/StreamGroupedFold.java | 8 +- .../api/operators/StreamGroupedReduce.java | 8 +- .../api/operators/StreamOperator.java | 6 +- .../operators/StreamingRuntimeContext.java | 80 +-- .../TwoInputTransformation.java | 47 ++ .../triggers/ContinuousEventTimeTrigger.java | 4 +- .../ContinuousProcessingTimeTrigger.java | 6 +- .../api/windowing/triggers/CountTrigger.java | 4 +- .../api/windowing/triggers/DeltaTrigger.java | 4 +- .../api/windowing/triggers/Trigger.java | 6 +- .../runtime/io/StreamInputProcessor.java | 2 +- .../runtime/io/StreamTwoInputProcessor.java | 2 + ...ctAlignedProcessingTimeWindowOperator.java | 4 +- .../windowing/NonKeyedWindowOperator.java | 17 +- .../operators/windowing/WindowOperator.java | 21 +- .../runtime/tasks/OperatorChain.java | 4 +- .../streaming/runtime/tasks/StreamTask.java | 58 +- .../runtime/tasks/StreamTaskState.java | 13 +- .../runtime/tasks/StreamTaskStateList.java | 4 +- .../flink/streaming/api/DataStreamTest.java | 10 +- .../api/operators/co/SelfConnectionTest.java | 6 +- ...ignedProcessingTimeWindowOperatorTest.java | 28 +- ...ignedProcessingTimeWindowOperatorTest.java | 68 +-- .../flink/streaming/util/MockContext.java | 33 +- .../OneInputStreamOperatorTestHarness.java | 44 +- .../TwoInputStreamOperatorTestHarness.java | 8 +- .../scala/StreamExecutionEnvironment.scala | 7 +- .../api/scala/function/StatefulFunction.scala | 2 +- ...EventTimeAllWindowCheckpointingITCase.java | 4 +- .../EventTimeWindowCheckpointingITCase.java | 4 +- .../PartitionedStateCheckpointingITCase.java | 26 +- .../test/checkpointing/SavepointITCase.java | 10 +- .../StateCheckpointedITCase.java | 8 +- .../StreamCheckpointNotifierITCase.java | 26 +- .../WindowCheckpointingITCase.java | 4 +- .../jar/CheckpointedStreamingProgram.java | 4 +- .../test/recovery/ChaosMonkeyITCase.java | 6 +- .../JobManagerCheckpointRecoveryITCase.java | 4 +- 109 files changed, 4540 insertions(+), 1802 deletions(-) rename flink-contrib/flink-streaming-contrib/src/main/java/org/apache/flink/contrib/streaming/state/{LazyDbKvState.java => LazyDbValueState.java} (77%) create mode 100644 flink-core/src/main/java/org/apache/flink/api/common/state/ListState.java create mode 100644 flink-core/src/main/java/org/apache/flink/api/common/state/ListStateDescriptor.java create mode 100644 flink-core/src/main/java/org/apache/flink/api/common/state/MergingState.java create mode 100644 flink-core/src/main/java/org/apache/flink/api/common/state/ReducingState.java create mode 100644 flink-core/src/main/java/org/apache/flink/api/common/state/ReducingStateDescriptor.java create mode 100644 flink-core/src/main/java/org/apache/flink/api/common/state/State.java create mode 100644 flink-core/src/main/java/org/apache/flink/api/common/state/StateBackend.java create mode 100644 flink-core/src/main/java/org/apache/flink/api/common/state/StateDescriptor.java create mode 100644 flink-core/src/main/java/org/apache/flink/api/common/state/ValueState.java create mode 100644 flink-core/src/main/java/org/apache/flink/api/common/state/ValueStateDescriptor.java delete mode 100644 flink-runtime/src/main/java/org/apache/flink/runtime/state/AbstractHeapKvState.java create mode 100644 flink-runtime/src/main/java/org/apache/flink/runtime/state/AbstractHeapState.java create mode 100644 flink-runtime/src/main/java/org/apache/flink/runtime/state/AbstractStateBackend.java create mode 100644 flink-runtime/src/main/java/org/apache/flink/runtime/state/ArrayListSerializer.java rename flink-streaming-java/src/main/java/org/apache/flink/streaming/api/checkpoint/CheckpointNotifier.java => flink-runtime/src/main/java/org/apache/flink/runtime/state/CheckpointListener.java (93%) create mode 100644 flink-runtime/src/main/java/org/apache/flink/runtime/state/GenericListState.java create mode 100644 flink-runtime/src/main/java/org/apache/flink/runtime/state/GenericReducingState.java delete mode 100644 flink-runtime/src/main/java/org/apache/flink/runtime/state/StateBackend.java rename flink-runtime/src/main/java/org/apache/flink/runtime/state/filesystem/{AbstractFileState.java => AbstractFileStateHandle.java} (95%) create mode 100644 flink-runtime/src/main/java/org/apache/flink/runtime/state/filesystem/AbstractFsState.java create mode 100644 flink-runtime/src/main/java/org/apache/flink/runtime/state/filesystem/AbstractFsStateSnapshot.java delete mode 100644 flink-runtime/src/main/java/org/apache/flink/runtime/state/filesystem/FsHeapKvState.java delete mode 100644 flink-runtime/src/main/java/org/apache/flink/runtime/state/filesystem/FsHeapKvStateSnapshot.java create mode 100644 flink-runtime/src/main/java/org/apache/flink/runtime/state/filesystem/FsListState.java create mode 100644 flink-runtime/src/main/java/org/apache/flink/runtime/state/filesystem/FsReducingState.java create mode 100644 flink-runtime/src/main/java/org/apache/flink/runtime/state/filesystem/FsValueState.java create mode 100644 flink-runtime/src/main/java/org/apache/flink/runtime/state/memory/AbstractMemState.java create mode 100644 flink-runtime/src/main/java/org/apache/flink/runtime/state/memory/AbstractMemStateSnapshot.java delete mode 100644 flink-runtime/src/main/java/org/apache/flink/runtime/state/memory/MemHeapKvState.java create mode 100644 flink-runtime/src/main/java/org/apache/flink/runtime/state/memory/MemListState.java create mode 100644 flink-runtime/src/main/java/org/apache/flink/runtime/state/memory/MemReducingState.java create mode 100644 flink-runtime/src/main/java/org/apache/flink/runtime/state/memory/MemValueState.java delete mode 100644 flink-runtime/src/main/java/org/apache/flink/runtime/state/memory/MemoryHeapKvStateSnapshot.java create mode 100644 flink-runtime/src/test/java/org/apache/flink/runtime/state/StateBackendTestBase.java diff --git a/flink-contrib/flink-streaming-contrib/src/main/java/org/apache/flink/contrib/streaming/state/DbStateBackend.java b/flink-contrib/flink-streaming-contrib/src/main/java/org/apache/flink/contrib/streaming/state/DbStateBackend.java index ad5ec56554912..c55b3c0e533e2 100644 --- a/flink-contrib/flink-streaming-contrib/src/main/java/org/apache/flink/contrib/streaming/state/DbStateBackend.java +++ b/flink-contrib/flink-streaming-contrib/src/main/java/org/apache/flink/contrib/streaming/state/DbStateBackend.java @@ -17,17 +17,26 @@ package org.apache.flink.contrib.streaming.state; -import java.io.IOException; import java.io.Serializable; import java.sql.Connection; import java.sql.PreparedStatement; import java.sql.SQLException; +import java.util.ArrayList; import java.util.Random; import java.util.concurrent.Callable; +import org.apache.flink.api.common.state.ListState; +import org.apache.flink.api.common.state.ListStateDescriptor; +import org.apache.flink.api.common.state.ReducingState; +import org.apache.flink.api.common.state.ReducingStateDescriptor; +import org.apache.flink.api.common.state.ValueState; +import org.apache.flink.api.common.state.ValueStateDescriptor; import org.apache.flink.api.common.typeutils.TypeSerializer; import org.apache.flink.runtime.execution.Environment; -import org.apache.flink.runtime.state.StateBackend; +import org.apache.flink.runtime.state.AbstractStateBackend; +import org.apache.flink.runtime.state.ArrayListSerializer; +import org.apache.flink.runtime.state.GenericListState; +import org.apache.flink.runtime.state.GenericReducingState; import org.apache.flink.runtime.state.StateHandle; import org.apache.flink.util.InstantiationUtil; import org.slf4j.Logger; @@ -36,9 +45,9 @@ import static org.apache.flink.contrib.streaming.state.SQLRetrier.retry; /** - * {@link StateBackend} for storing checkpoints in JDBC supporting databases. + * {@link AbstractStateBackend} for storing checkpoints in JDBC supporting databases. * Key-Value state is stored out-of-core and is lazily fetched using the - * {@link LazyDbKvState} implementation. A different backend can also be + * {@link LazyDbValueState} implementation. A different backend can also be * provided in the constructor to store the non-partitioned states. A common use * case would be to store the key-value states in the database and store larger * non-partitioned states on a distributed file system. @@ -56,7 +65,7 @@ * {@link MySqlAdapter} can be supplied in the {@link DbBackendConfig}. * */ -public class DbStateBackend extends StateBackend { +public class DbStateBackend extends AbstractStateBackend { private static final long serialVersionUID = 1L; private static final Logger LOG = LoggerFactory.getLogger(DbStateBackend.class); @@ -79,10 +88,12 @@ public class DbStateBackend extends StateBackend { private transient PreparedStatement insertStatement; + private String operatorIdentifier; + // ------------------------------------------------------ // We allow to use a different backend for storing non-partitioned states - private StateBackend nonPartitionedStateBackend = null; + private AbstractStateBackend nonPartitionedStateBackend = null; // ------------------------------------------------------ @@ -104,7 +115,7 @@ public DbStateBackend(DbBackendConfig backendConfig) { * non-partitioned state snapshots. * */ - public DbStateBackend(DbBackendConfig backendConfig, StateBackend backend) { + public DbStateBackend(DbBackendConfig backendConfig, AbstractStateBackend backend) { this(backendConfig); this.nonPartitionedStateBackend = backend; } @@ -160,7 +171,7 @@ public DbStateHandle call() throws Exception { insertStatement.executeUpdate(); - return new DbStateHandle(appIdShort, checkpointID, timestamp, handleId, + return new DbStateHandle<>(appIdShort, checkpointID, timestamp, handleId, dbConfig, serializedState.length); } }, numSqlRetries, sqlRetrySleep); @@ -182,20 +193,46 @@ public CheckpointStateOutputStream createCheckpointStateOutputStream(long checkp } @Override - public LazyDbKvState createKvState(String stateId, String stateName, - TypeSerializer keySerializer, TypeSerializer valueSerializer, V defaultValue) throws IOException { - return new LazyDbKvState( - stateId + "_" + env.getApplicationID().toShortString(), - env.getTaskInfo().getIndexOfThisSubtask() == 0, - getConnections(), - getConfiguration(), - keySerializer, - valueSerializer, - defaultValue); + protected ValueState createValueState(TypeSerializer namespaceSerializer, + ValueStateDescriptor stateDesc) throws Exception { + String stateName = operatorIdentifier + "_"+ stateDesc.getName(); + + return new LazyDbValueState<>( + stateName, + env.getTaskInfo().getIndexOfThisSubtask() == 0, + getConnections(), + getConfiguration(), + keySerializer, + namespaceSerializer, + stateDesc); + } + + @Override + protected ListState createListState(TypeSerializer namespaceSerializer, + ListStateDescriptor stateDesc) throws Exception { + ValueStateDescriptor> valueStateDescriptor = new ValueStateDescriptor<>(stateDesc.getName(), null, new ArrayListSerializer<>(stateDesc.getSerializer())); + ValueState> valueState = createValueState(namespaceSerializer, valueStateDescriptor); + return new GenericListState<>(valueState); + } + + @Override + @SuppressWarnings("unchecked") + protected ReducingState createReducingState(TypeSerializer namespaceSerializer, + ReducingStateDescriptor stateDesc) throws Exception { + + ValueStateDescriptor valueStateDescriptor = new ValueStateDescriptor<>(stateDesc.getName(), null, stateDesc.getSerializer()); + ValueState valueState = createValueState(namespaceSerializer, valueStateDescriptor); + return new GenericReducingState<>(valueState, stateDesc.getReduceFunction()); } @Override - public void initializeForJob(final Environment env) throws Exception { + public void initializeForJob(final Environment env, + String operatorIdentifier, + TypeSerializer keySerializer) throws Exception { + super.initializeForJob(env, operatorIdentifier, keySerializer); + + this.operatorIdentifier = operatorIdentifier; + this.rnd = new Random(); this.env = env; @@ -221,7 +258,7 @@ public PreparedStatement call() throws SQLException { } }, numSqlRetries, sqlRetrySleep); } else { - nonPartitionedStateBackend.initializeForJob(env); + nonPartitionedStateBackend.initializeForJob(env, operatorIdentifier, keySerializer); } if (LOG.isDebugEnabled()) { diff --git a/flink-contrib/flink-streaming-contrib/src/main/java/org/apache/flink/contrib/streaming/state/LazyDbKvState.java b/flink-contrib/flink-streaming-contrib/src/main/java/org/apache/flink/contrib/streaming/state/LazyDbValueState.java similarity index 77% rename from flink-contrib/flink-streaming-contrib/src/main/java/org/apache/flink/contrib/streaming/state/LazyDbKvState.java rename to flink-contrib/flink-streaming-contrib/src/main/java/org/apache/flink/contrib/streaming/state/LazyDbValueState.java index 5d16be6a8d61e..753850a570da3 100644 --- a/flink-contrib/flink-streaming-contrib/src/main/java/org/apache/flink/contrib/streaming/state/LazyDbKvState.java +++ b/flink-contrib/flink-streaming-contrib/src/main/java/org/apache/flink/contrib/streaming/state/LazyDbValueState.java @@ -19,6 +19,7 @@ import static org.apache.flink.contrib.streaming.state.SQLRetrier.retry; +import java.io.ByteArrayOutputStream; import java.io.IOException; import java.sql.Connection; import java.sql.SQLException; @@ -33,12 +34,15 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import org.apache.flink.api.common.state.ValueState; +import org.apache.flink.api.common.state.ValueStateDescriptor; import org.apache.flink.api.common.typeutils.TypeSerializer; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.contrib.streaming.state.ShardedConnection.ShardedStatement; +import org.apache.flink.core.memory.DataOutputViewStreamWrapper; import org.apache.flink.runtime.state.KvState; import org.apache.flink.runtime.state.KvStateSnapshot; -import org.apache.flink.streaming.api.checkpoint.CheckpointNotifier; +import org.apache.flink.runtime.state.CheckpointListener; import org.apache.flink.util.InstantiationUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -51,9 +55,10 @@ * cached on heap and are lazily retrieved on access. * */ -public class LazyDbKvState implements KvState, CheckpointNotifier { +public class LazyDbValueState + implements KvState, ValueStateDescriptor, DbStateBackend>, ValueState, CheckpointListener { - private static final Logger LOG = LoggerFactory.getLogger(LazyDbKvState.class); + private static final Logger LOG = LoggerFactory.getLogger(LazyDbValueState.class); // ------------------------------------------------------ @@ -62,10 +67,13 @@ public class LazyDbKvState implements KvState, Check private final boolean compact; private K currentKey; + private N currentNamespace; private final V defaultValue; private final TypeSerializer keySerializer; + private final TypeSerializer namespaceSerializer; private final TypeSerializer valueSerializer; + private final ValueStateDescriptor stateDesc; // ------------------------------------------------------ @@ -105,21 +113,31 @@ public class LazyDbKvState implements KvState, Check // ------------------------------------------------------ /** - * Constructor to initialize the {@link LazyDbKvState} the first time the + * Constructor to initialize the {@link LazyDbValueState} the first time the * job starts. */ - public LazyDbKvState(String kvStateId, boolean compact, ShardedConnection cons, DbBackendConfig conf, - TypeSerializer keySerializer, TypeSerializer valueSerializer, V defaultValue) throws IOException { - this(kvStateId, compact, cons, conf, keySerializer, valueSerializer, defaultValue, 1, 0); + public LazyDbValueState(String kvStateId, + boolean compact, + ShardedConnection cons, + DbBackendConfig conf, + TypeSerializer keySerializer, + TypeSerializer namespaceSerializer, + ValueStateDescriptor stateDesc) throws IOException { + this(kvStateId, compact, cons, conf, keySerializer, namespaceSerializer, stateDesc, 1, 0); } /** - * Initialize the {@link LazyDbKvState} from a snapshot. + * Initialize the {@link LazyDbValueState} from a snapshot. */ - public LazyDbKvState(String kvStateId, boolean compact, ShardedConnection cons, final DbBackendConfig conf, - TypeSerializer keySerializer, TypeSerializer valueSerializer, V defaultValue, long nextTs, - long lastCompactedTs) - throws IOException { + public LazyDbValueState(String kvStateId, + boolean compact, + ShardedConnection cons, + final DbBackendConfig conf, + TypeSerializer keySerializer, + TypeSerializer namespaceSerializer, + ValueStateDescriptor stateDesc, + long nextTs, + long lastCompactedTs) throws IOException { this.kvStateId = kvStateId; this.compact = compact; @@ -129,8 +147,10 @@ public LazyDbKvState(String kvStateId, boolean compact, ShardedConnection cons, } this.keySerializer = keySerializer; - this.valueSerializer = valueSerializer; - this.defaultValue = defaultValue; + this.namespaceSerializer = namespaceSerializer; + this.valueSerializer = stateDesc.getSerializer(); + this.defaultValue = stateDesc.getDefaultValue(); + this.stateDesc = stateDesc; this.maxInsertBatchSize = conf.getMaxKvInsertBatchSize(); this.conf = conf; @@ -159,10 +179,15 @@ public void setCurrentKey(K key) { this.currentKey = key; } + @Override + public void setCurrentNamespace(N namespace) { + this.currentNamespace = namespace; + } + @Override public void update(V value) throws IOException { try { - cache.put(currentKey, Optional.fromNullable(value)); + cache.put(Tuple2.of(currentKey, currentNamespace), Optional.fromNullable(value)); } catch (RuntimeException e) { // We need to catch the RuntimeExceptions thrown in the StateCache // methods here @@ -176,7 +201,7 @@ public V value() throws IOException { // We get the value from the cache (which will automatically load it // from the database if necessary). If null, we return a copy of the // default value - V val = cache.get(currentKey).orNull(); + V val = cache.get(Tuple2.of(currentKey, currentNamespace)).orNull(); return val != null ? val : copyDefault(); } catch (RuntimeException e) { // We need to catch the RuntimeExceptions thrown in the StateCache @@ -186,7 +211,12 @@ public V value() throws IOException { } @Override - public DbKvStateSnapshot snapshot(long checkpointId, long timestamp) throws IOException { + public void clear() { + cache.put(Tuple2.of(currentKey, currentNamespace), Optional.fromNullable(null)); + } + + @Override + public DbKvStateSnapshot snapshot(long checkpointId, long timestamp) throws IOException { // Validate timing assumptions if (timestamp <= nextTs) { @@ -198,7 +228,7 @@ public DbKvStateSnapshot snapshot(long checkpointId, long timestamp) throw if (!cache.modified.isEmpty()) { // We insert the modified elements to the database with the current // timestamp then clear the modified states - for (Entry> state : cache.modified.entrySet()) { + for (Entry, Optional> state : cache.modified.entrySet()) { batchInsert.add(state, timestamp); } batchInsert.flush(timestamp); @@ -219,14 +249,13 @@ public Void call() throws Exception { nextTs = timestamp + 1; completedCheckpoints.put(checkpointId, timestamp); - return new DbKvStateSnapshot(kvStateId, timestamp, lastCompactedTs); + return new DbKvStateSnapshot(kvStateId, timestamp, lastCompactedTs, namespaceSerializer, stateDesc); } /** * Returns the number of elements currently stored in the task's cache. Note * that the number of elements in the database is not counted here. */ - @Override public int size() { return cache.size(); } @@ -299,7 +328,7 @@ public void dispose() { * Return the Map of cached states. * */ - public Map> getStateCache() { + public Map, Optional> getStateCache() { return cache; } @@ -308,7 +337,7 @@ public Map> getStateCache() { * database yet. * */ - public Map> getModified() { + public Map, Optional> getModified() { return cache.modified; } @@ -333,7 +362,7 @@ public ExecutorService getExecutor() { * checkpoint and recovery timestamp. * */ - private static class DbKvStateSnapshot implements KvStateSnapshot { + private static class DbKvStateSnapshot implements KvStateSnapshot, ValueStateDescriptor, DbStateBackend> { private static final long serialVersionUID = 1L; @@ -341,16 +370,30 @@ private static class DbKvStateSnapshot implements KvStateSnapshot namespaceSerializer; + + /** StateDescriptor, for sanity checks */ + private final ValueStateDescriptor stateDesc; + + public DbKvStateSnapshot(String kvStateId, + long checkpointTimestamp, + long lastCompactedTs, + TypeSerializer namespaceSerializer, + ValueStateDescriptor stateDesc) { this.checkpointTimestamp = checkpointTimestamp; this.kvStateId = kvStateId; this.lastCompactedTimestamp = lastCompactedTs; + this.namespaceSerializer = namespaceSerializer; + this.stateDesc = stateDesc; } @Override - public LazyDbKvState restoreState(final DbStateBackend stateBackend, - final TypeSerializer keySerializer, final TypeSerializer valueSerializer, final V defaultValue, - ClassLoader classLoader, final long recoveryTimestamp) throws IOException { + public KvState, ValueStateDescriptor, DbStateBackend> restoreState( + final DbStateBackend stateBackend, + TypeSerializer keySerializer, + ClassLoader classLoader, + final long recoveryTimestamp) throws Exception { // Validate timing assumptions if (recoveryTimestamp <= checkpointTimestamp) { @@ -379,9 +422,15 @@ public Void call() throws Exception { boolean cleanup = stateBackend.getEnvironment().getTaskInfo().getIndexOfThisSubtask() == 0; // Restore the KvState - LazyDbKvState restored = new LazyDbKvState(kvStateId, cleanup, - stateBackend.getConnections(), stateBackend.getConfiguration(), keySerializer, valueSerializer, - defaultValue, recoveryTimestamp, lastCompactedTimestamp); + LazyDbValueState restored = new LazyDbValueState<>(kvStateId, + cleanup, + stateBackend.getConnections(), + stateBackend.getConfiguration(), + keySerializer, + namespaceSerializer, + stateDesc, + recoveryTimestamp, + lastCompactedTimestamp); if (LOG.isDebugEnabled()) { LOG.debug("KV state({},{}) restored.", kvStateId, recoveryTimestamp); @@ -412,14 +461,14 @@ public long getStateSize() throws Exception { * Keys not found in the cached will be retrieved from the underlying * database */ - private final class StateCache extends LinkedHashMap> { + private final class StateCache extends LinkedHashMap, Optional> { private static final long serialVersionUID = 1L; private final int cacheSize; private final int evictionSize; // We keep track the state modified since the last checkpoint - private final Map> modified = new HashMap<>(); + private final Map, Optional> modified = new HashMap<>(); public StateCache(int cacheSize, int evictionSize) { super(cacheSize, 0.75f, true); @@ -428,7 +477,7 @@ public StateCache(int cacheSize, int evictionSize) { } @Override - public Optional put(K key, Optional value) { + public Optional put(Tuple2 key, Optional value) { // Put kv pair in the cache and evict elements if the cache is full Optional old = super.put(key, value); modified.put(key, value); @@ -443,14 +492,14 @@ public Optional get(Object key) { Optional value = super.get(key); if (value == null) { // If it doesn't try to load it from the database - value = Optional.fromNullable(getFromDatabaseOrNull((K) key)); - put((K) key, value); + value = Optional.fromNullable(getFromDatabaseOrNull((Tuple2) key)); + put((Tuple2) key, value); } return value; } @Override - protected boolean removeEldestEntry(Entry> eldest) { + protected boolean removeEldestEntry(Entry, Optional> eldest) { // We need to remove elements manually if the cache becomes full, so // we always return false here. return false; @@ -463,15 +512,20 @@ protected boolean removeEldestEntry(Entry> eldest) { * @return The value corresponding to the key and the last checkpointid * from the database if exists or null. */ - private V getFromDatabaseOrNull(final K key) { + private V getFromDatabaseOrNull(final Tuple2 key) { try { return retry(new Callable() { public V call() throws Exception { - byte[] serializedKey = InstantiationUtil.serializeToByteArray(keySerializer, key); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + DataOutputViewStreamWrapper out = new DataOutputViewStreamWrapper(baos); + keySerializer.serialize(key.f0, out); + namespaceSerializer.serialize(key.f1, out); + out.close(); + // We lookup using the adapter and serialize/deserialize // with the TypeSerializers byte[] serializedVal = dbAdapter.lookupKey(kvStateId, - selectStatements.getForKey(key), serializedKey, nextTs); + selectStatements.getForKey(key.f0), baos.toByteArray(), nextTs); return serializedVal != null ? InstantiationUtil.deserializeFromByteArray(valueSerializer, serializedVal) : null; @@ -497,10 +551,10 @@ private void evictIfFull() { try { int numEvicted = 0; - Iterator>> entryIterator = entrySet().iterator(); + Iterator, Optional>> entryIterator = entrySet().iterator(); while (numEvicted++ < evictionSize && entryIterator.hasNext()) { - Entry> next = entryIterator.next(); + Entry, Optional> next = entryIterator.next(); // We only need to write to the database if modified if (modified.remove(next.getKey()) != null) { @@ -522,7 +576,7 @@ private void evictIfFull() { } @Override - public void putAll(Map> m) { + public void putAll(Map, ? extends Optional> m) { throw new UnsupportedOperationException(); } @@ -553,9 +607,10 @@ public BatchInserter(int numShards) { } } - public void add(Entry> next, long timestamp) throws IOException { + public void add(Entry, Optional> next, long timestamp) throws IOException { - K key = next.getKey(); + K key = next.getKey().f0; + N namespace = next.getKey().f1; V value = next.getValue().orNull(); // Get the current partition if present or initialize empty list @@ -564,9 +619,15 @@ public void add(Entry> next, long timestamp) throws IOException { List> insertPartition = inserts[shardIndex]; // Add the k-v pair to the partition - byte[] k = InstantiationUtil.serializeToByteArray(keySerializer, key); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + DataOutputViewStreamWrapper out = new DataOutputViewStreamWrapper(baos); + keySerializer.serialize(key, out); + namespaceSerializer.serialize(namespace, out); + out.close(); + + byte[] kn = baos.toByteArray(); byte[] v = value != null ? InstantiationUtil.serializeToByteArray(valueSerializer, value) : null; - insertPartition.add(Tuple2.of(k, v)); + insertPartition.add(Tuple2.of(kn, v)); // If partition is full write to the database and clear if (insertPartition.size() == maxInsertBatchSize) { diff --git a/flink-contrib/flink-streaming-contrib/src/main/java/org/apache/flink/contrib/streaming/state/MySqlAdapter.java b/flink-contrib/flink-streaming-contrib/src/main/java/org/apache/flink/contrib/streaming/state/MySqlAdapter.java index 9eb3cd53f8d07..cf2b5be4e5621 100644 --- a/flink-contrib/flink-streaming-contrib/src/main/java/org/apache/flink/contrib/streaming/state/MySqlAdapter.java +++ b/flink-contrib/flink-streaming-contrib/src/main/java/org/apache/flink/contrib/streaming/state/MySqlAdapter.java @@ -195,7 +195,7 @@ public void compactKvStates(String stateId, Connection con, long lowerId, long u */ protected static void validateStateId(String name) { if (!name.matches("[a-zA-Z0-9_]+")) { - throw new RuntimeException("State name contains invalid characters."); + throw new RuntimeException("State name contains invalid characters: " + name); } } diff --git a/flink-contrib/flink-streaming-contrib/src/test/java/org/apache/flink/contrib/streaming/state/DbStateBackendTest.java b/flink-contrib/flink-streaming-contrib/src/test/java/org/apache/flink/contrib/streaming/state/DbStateBackendTest.java index 155ced8dd98fd..34adf75e6109a 100644 --- a/flink-contrib/flink-streaming-contrib/src/test/java/org/apache/flink/contrib/streaming/state/DbStateBackendTest.java +++ b/flink-contrib/flink-streaming-contrib/src/test/java/org/apache/flink/contrib/streaming/state/DbStateBackendTest.java @@ -18,12 +18,6 @@ package org.apache.flink.contrib.streaming.state; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertTrue; - import java.io.File; import java.io.IOException; import java.net.InetAddress; @@ -39,10 +33,20 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; +import com.google.common.base.Joiner; import org.apache.commons.io.FileUtils; import org.apache.derby.drda.NetworkServerControl; +import org.apache.flink.api.common.functions.ReduceFunction; +import org.apache.flink.api.common.state.ListState; +import org.apache.flink.api.common.state.ListStateDescriptor; +import org.apache.flink.api.common.state.ReducingState; +import org.apache.flink.api.common.state.ReducingStateDescriptor; +import org.apache.flink.api.common.state.ValueState; +import org.apache.flink.api.common.state.ValueStateDescriptor; import org.apache.flink.api.common.typeutils.base.IntSerializer; import org.apache.flink.api.common.typeutils.base.StringSerializer; +import org.apache.flink.api.common.typeutils.base.VoidSerializer; +import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.configuration.ConfigConstants; import org.apache.flink.core.testutils.CommonTestUtils; import org.apache.flink.runtime.execution.Environment; @@ -58,6 +62,9 @@ import com.google.common.base.Optional; +import static org.junit.Assert.*; +import static org.junit.Assert.fail; + public class DbStateBackendTest { private static NetworkServerControl server; @@ -129,7 +136,7 @@ public void testSetupAndSerialization() throws Exception { assertFalse(backend.isInitialized()); Environment env = new DummyEnvironment("test", 1, 0); - backend.initializeForJob(env); + backend.initializeForJob(env, "dummy-setup-ser", StringSerializer.INSTANCE); assertNotNull(backend.getConnections()); assertTrue( @@ -148,7 +155,7 @@ public void testSerializableState() throws Exception { Environment env = new DummyEnvironment("test", 1, 0); DbStateBackend backend = new DbStateBackend(conf); - backend.initializeForJob(env); + backend.initializeForJob(env, "dummy-ser-state", StringSerializer.INSTANCE); String state1 = "dummy state"; String state2 = "row row row your boat"; @@ -191,16 +198,20 @@ public void testKeyValueState() throws Exception { Environment env = new DummyEnvironment("test", 2, 0); - backend.initializeForJob(env); + backend.initializeForJob(env, "dummy_test_kv", IntSerializer.INSTANCE); + + ValueState state = backend.createValueState(IntSerializer.INSTANCE, + new ValueStateDescriptor<>("state1", null, StringSerializer.INSTANCE)); - LazyDbKvState kv = backend.createKvState("state1_1", "state1", IntSerializer.INSTANCE, - StringSerializer.INSTANCE, null); + LazyDbValueState kv = (LazyDbValueState) state; - String tableName = "state1_1_" + env.getApplicationID().toShortString(); + String tableName = "dummy_test_kv_state1"; assertTrue(isTableCreated(backend.getConnections().getFirst(), tableName)); assertEquals(0, kv.size()); + kv.setCurrentNamespace(1); + // some modifications to the state kv.setCurrentKey(1); assertNull(kv.value()); @@ -225,7 +236,7 @@ public void testKeyValueState() throws Exception { kv.update("u3"); // draw another snapshot - KvStateSnapshot snapshot2 = kv.snapshot(682375462379L, + KvStateSnapshot, ValueStateDescriptor, DbStateBackend> snapshot2 = kv.snapshot(682375462379L, 200); // validate the original state @@ -238,16 +249,23 @@ public void testKeyValueState() throws Exception { assertEquals("u3", kv.value()); // restore the first snapshot and validate it - KvState restored2 = snapshot2.restoreState(backend, IntSerializer.INSTANCE, - StringSerializer.INSTANCE, null, getClass().getClassLoader(), 6823754623710L); + KvState, ValueStateDescriptor, DbStateBackend> restored2 = snapshot2.restoreState( + backend, + IntSerializer.INSTANCE, + getClass().getClassLoader(), + 6823754623710L); + + restored2.setCurrentNamespace(1); + + @SuppressWarnings("unchecked") + ValueState restoredState2 = (ValueState) restored2; - assertEquals(0, restored2.size()); restored2.setCurrentKey(1); - assertEquals("u1", restored2.value()); + assertEquals("u1", restoredState2.value()); restored2.setCurrentKey(2); - assertEquals("u2", restored2.value()); + assertEquals("u2", restoredState2.value()); restored2.setCurrentKey(3); - assertEquals("u3", restored2.value()); + assertEquals("u3", restoredState2.value()); backend.close(); } finally { @@ -255,6 +273,173 @@ public void testKeyValueState() throws Exception { } } + @Test + @SuppressWarnings("unchecked,rawtypes") + public void testListState() { + File tempDir = new File(ConfigConstants.DEFAULT_TASK_MANAGER_TMP_PATH, UUID.randomUUID().toString()); + try { + FsStateBackend fileBackend = new FsStateBackend(localFileUri(tempDir)); + + DbStateBackend backend = new DbStateBackend(conf, fileBackend); + + Environment env = new DummyEnvironment("test", 2, 0); + + backend.initializeForJob(env, "dummy_test_kv_list", IntSerializer.INSTANCE); + + ListStateDescriptor kvId = new ListStateDescriptor<>("id", StringSerializer.INSTANCE); + ListState state = backend.getPartitionedState(null, VoidSerializer.INSTANCE, kvId); + + @SuppressWarnings("unchecked") + KvState, ListStateDescriptor, DbStateBackend> kv = + (KvState, ListStateDescriptor, DbStateBackend>) state; + + Joiner joiner = Joiner.on(","); + // some modifications to the state + kv.setCurrentKey(1); + assertEquals("", joiner.join(state.get())); + state.add("1"); + kv.setCurrentKey(2); + assertEquals("", joiner.join(state.get())); + state.add("2"); + kv.setCurrentKey(1); + assertEquals("1", joiner.join(state.get())); + + // draw a snapshot + KvStateSnapshot, ListStateDescriptor, DbStateBackend> snapshot1 = + kv.snapshot(682375462378L, 2); + + // make some more modifications + kv.setCurrentKey(1); + state.add("u1"); + kv.setCurrentKey(2); + state.add("u2"); + kv.setCurrentKey(3); + state.add("u3"); + + // draw another snapshot + KvStateSnapshot, ListStateDescriptor, DbStateBackend> snapshot2 = + kv.snapshot(682375462379L, 4); + + // validate the original state + kv.setCurrentKey(1); + assertEquals("1,u1", joiner.join(state.get())); + kv.setCurrentKey(2); + assertEquals("2,u2", joiner.join(state.get())); + kv.setCurrentKey(3); + assertEquals("u3", joiner.join(state.get())); + + kv.dispose(); + + // restore the second snapshot and validate it + KvState, ListStateDescriptor, DbStateBackend> restored2 = snapshot2.restoreState( + backend, + IntSerializer.INSTANCE, + this.getClass().getClassLoader(), 20); + + @SuppressWarnings("unchecked") + ListState restored2State = (ListState) restored2; + + restored2.setCurrentKey(1); + assertEquals("1,u1", joiner.join(restored2State.get())); + restored2.setCurrentKey(2); + assertEquals("2,u2", joiner.join(restored2State.get())); + restored2.setCurrentKey(3); + assertEquals("u3", joiner.join(restored2State.get())); + } + catch (Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + @Test + @SuppressWarnings("unchecked,rawtypes") + public void testReducingState() { + File tempDir = new File(ConfigConstants.DEFAULT_TASK_MANAGER_TMP_PATH, UUID.randomUUID().toString()); + try { + FsStateBackend fileBackend = new FsStateBackend(localFileUri(tempDir)); + + DbStateBackend backend = new DbStateBackend(conf, fileBackend); + + Environment env = new DummyEnvironment("test", 2, 0); + + backend.initializeForJob(env, "dummy_test_kv_reduce", IntSerializer.INSTANCE); + + ReducingStateDescriptor kvId = new ReducingStateDescriptor<>("id", + new ReduceFunction() { + private static final long serialVersionUID = 1L; + + @Override + public String reduce(String value1, String value2) throws Exception { + return value1 + "," + value2; + } + }, + StringSerializer.INSTANCE); + ReducingState state = backend.getPartitionedState(null, VoidSerializer.INSTANCE, kvId); + + @SuppressWarnings("unchecked") + KvState, ReducingStateDescriptor, DbStateBackend> kv = + (KvState, ReducingStateDescriptor, DbStateBackend>) state; + + Joiner joiner = Joiner.on(","); + // some modifications to the state + kv.setCurrentKey(1); + assertEquals(null, state.get()); + state.add("1"); + kv.setCurrentKey(2); + assertEquals(null, state.get()); + state.add("2"); + kv.setCurrentKey(1); + assertEquals("1", state.get()); + + // draw a snapshot + KvStateSnapshot, ReducingStateDescriptor, DbStateBackend> snapshot1 = + kv.snapshot(682375462378L, 2); + + // make some more modifications + kv.setCurrentKey(1); + state.add("u1"); + kv.setCurrentKey(2); + state.add("u2"); + kv.setCurrentKey(3); + state.add("u3"); + + // draw another snapshot + KvStateSnapshot, ReducingStateDescriptor, DbStateBackend> snapshot2 = + kv.snapshot(682375462379L, 4); + + // validate the original state + kv.setCurrentKey(1); + assertEquals("1,u1", state.get()); + kv.setCurrentKey(2); + assertEquals("2,u2", state.get()); + kv.setCurrentKey(3); + assertEquals("u3", state.get()); + + kv.dispose(); + + // restore the second snapshot and validate it + KvState, ReducingStateDescriptor, DbStateBackend> restored2 = snapshot2.restoreState( + backend, + IntSerializer.INSTANCE, + this.getClass().getClassLoader(), 20); + + @SuppressWarnings("unchecked") + ReducingState restored2State = (ReducingState) restored2; + + restored2.setCurrentKey(1); + assertEquals("1,u1", restored2State.get()); + restored2.setCurrentKey(2); + assertEquals("2,u2", restored2State.get()); + restored2.setCurrentKey(3); + assertEquals("u3", restored2State.get()); + } + catch (Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + @Test public void testCompaction() throws Exception { DbBackendConfig conf = new DbBackendConfig("flink", "flink", url1); @@ -266,13 +451,17 @@ public void testCompaction() throws Exception { DbStateBackend backend2 = new DbStateBackend(conf); DbStateBackend backend3 = new DbStateBackend(conf); - backend1.initializeForJob(new DummyEnvironment("test", 3, 0)); - backend2.initializeForJob(new DummyEnvironment("test", 3, 1)); - backend3.initializeForJob(new DummyEnvironment("test", 3, 2)); + backend1.initializeForJob(new DummyEnvironment("test", 3, 0), "dummy_1", StringSerializer.INSTANCE); + backend2.initializeForJob(new DummyEnvironment("test", 3, 1), "dummy_2", StringSerializer.INSTANCE); + backend3.initializeForJob(new DummyEnvironment("test", 3, 2), "dummy_3", StringSerializer.INSTANCE); - LazyDbKvState s1 = backend1.createKvState("a_1", "a", null, null, null); - LazyDbKvState s2 = backend2.createKvState("a_1", "a", null, null, null); - LazyDbKvState s3 = backend3.createKvState("a_1", "a", null, null, null); + ValueState s1State = backend1.createValueState(StringSerializer.INSTANCE, new ValueStateDescriptor<>("a1", null, StringSerializer.INSTANCE)); + ValueState s2State = backend2.createValueState(StringSerializer.INSTANCE, new ValueStateDescriptor<>("a2", null, StringSerializer.INSTANCE)); + ValueState s3State = backend3.createValueState(StringSerializer.INSTANCE, new ValueStateDescriptor<>("a3", null, StringSerializer.INSTANCE)); + + LazyDbValueState s1 = (LazyDbValueState) s1State; + LazyDbValueState s2 = (LazyDbValueState) s2State; + LazyDbValueState s3 = (LazyDbValueState) s3State; assertTrue(s1.isCompactor()); assertFalse(s2.isCompactor()); @@ -324,23 +513,27 @@ public void testCaching() throws Exception { Environment env = new DummyEnvironment("test", 2, 0); - String tableName = "state1_1_" + env.getApplicationID().toShortString(); + String tableName = "dummy_test_caching_state1"; assertFalse(isTableCreated(DriverManager.getConnection(url1, "flink", "flink"), tableName)); assertFalse(isTableCreated(DriverManager.getConnection(url2, "flink", "flink"), tableName)); - backend.initializeForJob(env); + backend.initializeForJob(env, "dummy_test_caching", IntSerializer.INSTANCE); + + ValueState state = backend.createValueState(IntSerializer.INSTANCE, + new ValueStateDescriptor<>("state1", "a", StringSerializer.INSTANCE)); - LazyDbKvState kv = backend.createKvState("state1_1", "state1", IntSerializer.INSTANCE, - StringSerializer.INSTANCE, "a"); + LazyDbValueState kv = (LazyDbValueState) state; assertTrue(isTableCreated(DriverManager.getConnection(url1, "flink", "flink"), tableName)); assertTrue(isTableCreated(DriverManager.getConnection(url2, "flink", "flink"), tableName)); - Map> cache = kv.getStateCache(); - Map> modified = kv.getModified(); + Map, Optional> cache = kv.getStateCache(); + Map, Optional> modified = kv.getModified(); assertEquals(0, kv.size()); + kv.setCurrentNamespace(1); + // some modifications to the state kv.setCurrentKey(1); assertEquals("a", kv.value()); @@ -360,24 +553,24 @@ public void testCaching() throws Exception { kv.update("3"); assertEquals("3", kv.value()); - assertTrue(modified.containsKey(1)); - assertTrue(modified.containsKey(2)); - assertTrue(modified.containsKey(3)); + assertTrue(modified.containsKey(Tuple2.of(1, 1))); + assertTrue(modified.containsKey(Tuple2.of(2, 1))); + assertTrue(modified.containsKey(Tuple2.of(3, 1))); // 1,2 should be evicted as the cache filled kv.setCurrentKey(4); kv.update("4"); assertEquals("4", kv.value()); - assertFalse(modified.containsKey(1)); - assertFalse(modified.containsKey(2)); - assertTrue(modified.containsKey(3)); - assertTrue(modified.containsKey(4)); + assertFalse(modified.containsKey(Tuple2.of(1, 1))); + assertFalse(modified.containsKey(Tuple2.of(2, 1))); + assertTrue(modified.containsKey(Tuple2.of(3, 1))); + assertTrue(modified.containsKey(Tuple2.of(4, 1))); - assertEquals(Optional.of("3"), cache.get(3)); - assertEquals(Optional.of("4"), cache.get(4)); - assertFalse(cache.containsKey(1)); - assertFalse(cache.containsKey(2)); + assertEquals(Optional.of("3"), cache.get(Tuple2.of(3, 1))); + assertEquals(Optional.of("4"), cache.get(Tuple2.of(4, 1))); + assertFalse(cache.containsKey(Tuple2.of(1, 1))); + assertFalse(cache.containsKey(Tuple2.of(2, 1))); // draw a snapshot kv.snapshot(682375462378L, 100); @@ -390,19 +583,19 @@ public void testCaching() throws Exception { kv.update(null); assertEquals("a", kv.value()); - assertTrue(modified.containsKey(2)); + assertTrue(modified.containsKey(Tuple2.of(2, 1))); assertEquals(1, modified.size()); - assertEquals(Optional.of("3"), cache.get(3)); - assertEquals(Optional.of("4"), cache.get(4)); - assertEquals(Optional.absent(), cache.get(2)); - assertFalse(cache.containsKey(1)); + assertEquals(Optional.of("3"), cache.get(Tuple2.of(3, 1))); + assertEquals(Optional.of("4"), cache.get(Tuple2.of(4, 1))); + assertEquals(Optional.absent(), cache.get(Tuple2.of(2, 1))); + assertFalse(cache.containsKey(Tuple2.of(1, 1))); - assertTrue(modified.containsKey(2)); - assertFalse(modified.containsKey(3)); - assertFalse(modified.containsKey(4)); - assertTrue(cache.containsKey(3)); - assertTrue(cache.containsKey(4)); + assertTrue(modified.containsKey(Tuple2.of(2, 1))); + assertFalse(modified.containsKey(Tuple2.of(3, 1))); + assertFalse(modified.containsKey(Tuple2.of(4, 1))); + assertTrue(cache.containsKey(Tuple2.of(3, 1))); + assertTrue(cache.containsKey(Tuple2.of(4, 1))); // clear cache from initial keys @@ -413,14 +606,14 @@ public void testCaching() throws Exception { kv.setCurrentKey(7); kv.value(); - assertFalse(modified.containsKey(5)); - assertTrue(modified.containsKey(6)); - assertTrue(modified.containsKey(7)); + assertFalse(modified.containsKey(Tuple2.of(5, 1))); + assertTrue(modified.containsKey(Tuple2.of(6, 1))); + assertTrue(modified.containsKey(Tuple2.of(7, 1))); - assertFalse(cache.containsKey(1)); - assertFalse(cache.containsKey(2)); - assertFalse(cache.containsKey(3)); - assertFalse(cache.containsKey(4)); + assertFalse(cache.containsKey(Tuple2.of(1, 1))); + assertFalse(cache.containsKey(Tuple2.of(2, 1))); + assertFalse(cache.containsKey(Tuple2.of(3, 1))); + assertFalse(cache.containsKey(Tuple2.of(4, 1))); kv.setCurrentKey(2); assertEquals("a", kv.value()); @@ -428,7 +621,8 @@ public void testCaching() throws Exception { long checkpointTs = System.currentTimeMillis(); // Draw a snapshot that we will restore later - KvStateSnapshot snapshot1 = kv.snapshot(682375462379L, checkpointTs); + KvStateSnapshot, ValueStateDescriptor, DbStateBackend> snapshot1 = kv.snapshot(682375462379L, checkpointTs); + assertTrue(modified.isEmpty()); // Do some updates then draw another snapshot (imitate a partial @@ -448,17 +642,35 @@ public void testCaching() throws Exception { // restore the second snapshot and validate it (we set a new default // value here to make sure that the default wasn't written) - KvState restored = snapshot1.restoreState(backend, IntSerializer.INSTANCE, - StringSerializer.INSTANCE, "b", getClass().getClassLoader(), 6823754623711L); + KvState, ValueStateDescriptor, DbStateBackend> restored = snapshot1.restoreState( + backend, + IntSerializer.INSTANCE, + getClass().getClassLoader(), + 6823754623711L); + + LazyDbValueState lazyRestored = (LazyDbValueState) restored; + + cache = lazyRestored.getStateCache(); + modified = lazyRestored.getModified(); + + restored.setCurrentNamespace(1); + + @SuppressWarnings("unchecked") + ValueState restoredState = (ValueState) restored; restored.setCurrentKey(1); - assertEquals("b", restored.value()); + + assertEquals("a", restoredState.value()); + // make sure that we got the default and not some value from the db + assertEquals(cache.get(Tuple2.of(1, 1)), Optional.absent()); restored.setCurrentKey(2); - assertEquals("b", restored.value()); + assertEquals("a", restoredState.value()); + // make sure that we got the default and not some value from the db + assertEquals(cache.get(Tuple2.of(2, 1)), Optional.absent()); restored.setCurrentKey(3); - assertEquals("3", restored.value()); + assertEquals("3", restoredState.value()); restored.setCurrentKey(4); - assertEquals("4", restored.value()); + assertEquals("4", restoredState.value()); backend.close(); } diff --git a/flink-contrib/flink-streaming-contrib/src/test/java/org/apache/flink/contrib/streaming/state/DerbyAdapter.java b/flink-contrib/flink-streaming-contrib/src/test/java/org/apache/flink/contrib/streaming/state/DerbyAdapter.java index 02c1a3ede009b..53d8d503f909d 100644 --- a/flink-contrib/flink-streaming-contrib/src/test/java/org/apache/flink/contrib/streaming/state/DerbyAdapter.java +++ b/flink-contrib/flink-streaming-contrib/src/test/java/org/apache/flink/contrib/streaming/state/DerbyAdapter.java @@ -69,7 +69,6 @@ public void createCheckpointsTable(String appId, Connection con) throws SQLExcep */ @Override public void createKVStateTable(String stateId, Connection con) throws SQLException { - validateStateId(stateId); try (Statement smt = con.createStatement()) { smt.executeUpdate( diff --git a/flink-contrib/flink-streaming-contrib/src/test/resources/log4j-test.properties b/flink-contrib/flink-streaming-contrib/src/test/resources/log4j-test.properties index 0b686e543bb23..45a18ec404caf 100644 --- a/flink-contrib/flink-streaming-contrib/src/test/resources/log4j-test.properties +++ b/flink-contrib/flink-streaming-contrib/src/test/resources/log4j-test.properties @@ -17,11 +17,11 @@ ################################################################################ # Set root logger level to DEBUG and its only appender to A1. -log4j.rootLogger=OFF, A1 +log4j.rootLogger=ON, A1 # A1 is set to be a ConsoleAppender. log4j.appender.A1=org.apache.log4j.ConsoleAppender # A1 uses PatternLayout. log4j.appender.A1.layout=org.apache.log4j.PatternLayout -log4j.appender.A1.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n \ No newline at end of file +log4j.appender.A1.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n diff --git a/flink-core/src/main/java/org/apache/flink/api/common/functions/RuntimeContext.java b/flink-core/src/main/java/org/apache/flink/api/common/functions/RuntimeContext.java index a1e9d7d0cfe35..a419d1e978c34 100644 --- a/flink-core/src/main/java/org/apache/flink/api/common/functions/RuntimeContext.java +++ b/flink-core/src/main/java/org/apache/flink/api/common/functions/RuntimeContext.java @@ -31,7 +31,9 @@ import org.apache.flink.api.common.accumulators.IntCounter; import org.apache.flink.api.common.accumulators.LongCounter; import org.apache.flink.api.common.cache.DistributedCache; -import org.apache.flink.api.common.state.OperatorState; +import org.apache.flink.api.common.state.ValueState; +import org.apache.flink.api.common.state.State; +import org.apache.flink.api.common.state.StateDescriptor; import org.apache.flink.api.common.typeinfo.TypeInformation; /** @@ -186,9 +188,59 @@ public interface RuntimeContext { // -------------------------------------------------------------------------------------------- + /** + * Gets the partitioned state, which is only accessible if the function is executed on + * a KeyedStream. When interacting with the state only the instance bound to the key of the + * element currently processed by the function is changed. + * Each operator may maintain multiple partitioned states, addressed with different names. + * + *

Because the scope of each value is the key of the currently processed element, + * and the elements are distributed by the Flink runtime, the system can transparently + * scale out and redistribute the state and KeyedStream. + * + *

The following code example shows how to implement a continuous counter that counts + * how many times elements of a certain key occur, and emits an updated count for that + * element on each occurrence. + * + * 

{@code
+	 * DataStream stream = ...;
+	 * KeyedStream keyedStream = stream.keyBy("id");
+	 *
+	 * keyedStream.map(new RichMapFunction>() {
+	 *
+	 *     private ValueStateDescriptor countIdentifier =
+	 *         new ValueStateDescriptor<>("count", 0L, LongSerializer.INSTANCE);
+	 *
+	 *     private ValueState count;
+	 *
+	 *     public void open(Configuration cfg) {
+	 *         state = getRuntimeContext().getPartitionedState(countIdentifier);
+	 *     }
+	 *
+	 *     public Tuple2 map(MyType value) {
+	 *         long count = state.value();
+	 *         state.update(value + 1);
+	 *         return new Tuple2<>(value, count);
+	 *     }
+	 * });
+	 *
+	 * }
+ * + * @param stateDescriptor The StateDescriptor that contains the name and type of the + * state that is being accessed. + * + * @param The type of the state. + * + * @return The partitioned state object. + * + * @throws UnsupportedOperationException Thrown, if no partitioned state is available for the + * function (function is not part os a KeyedStream). + */ + S getPartitionedState(StateDescriptor stateDescriptor); + /** * Gets the key/value state, which is only accessible if the function is executed on - * a KeyedStream. Upon calling {@link OperatorState#value()}, the key/value state will + * a KeyedStream. Upon calling {@link ValueState#value()}, the key/value state will * return the value bound to the key of the element currently processed by the function. * Each operator may maintain multiple key/value states, addressed with different names. * @@ -226,11 +278,13 @@ public interface RuntimeContext { * the TypeInformation object must be manually passed via * {@link #getKeyValueState(String, TypeInformation, Object)}. * + * * @param name The name of the key/value state. * @param stateType The class of the type that is stored in the state. Used to generate * serializers for managed memory and checkpointing. * @param defaultState The default state value, returned when the state is accessed and * no value has yet been set for the key. May be null. + * * @param The type of the state. * * @return The key/value state access. @@ -238,11 +292,12 @@ public interface RuntimeContext { * @throws UnsupportedOperationException Thrown, if no key/value state is available for the * function (function is not part os a KeyedStream). */ - OperatorState getKeyValueState(String name, Class stateType, S defaultState); + @Deprecated + ValueState getKeyValueState(String name, Class stateType, S defaultState); /** * Gets the key/value state, which is only accessible if the function is executed on - * a KeyedStream. Upon calling {@link OperatorState#value()}, the key/value state will + * a KeyedStream. Upon calling {@link ValueState#value()}, the key/value state will * return the value bound to the key of the element currently processed by the function. * Each operator may maintain multiple key/value states, addressed with different names. * @@ -275,17 +330,19 @@ public interface RuntimeContext { * * } * + * * @param name The name of the key/value state. * @param stateType The type information for the type that is stored in the state. - * Used to create serializers for managed memory and checkpoints. + * Used to create serializers for managed memory and checkpoints. * @param defaultState The default state value, returned when the state is accessed and * no value has yet been set for the key. May be null. * @param The type of the state. - * + * * @return The key/value state access. * * @throws UnsupportedOperationException Thrown, if no key/value state is available for the * function (function is not part os a KeyedStream). */ - OperatorState getKeyValueState(String name, TypeInformation stateType, S defaultState); + @Deprecated + ValueState getKeyValueState(String name, TypeInformation stateType, S defaultState); } diff --git a/flink-core/src/main/java/org/apache/flink/api/common/functions/util/AbstractRuntimeUDFContext.java b/flink-core/src/main/java/org/apache/flink/api/common/functions/util/AbstractRuntimeUDFContext.java index 8f1b6b1429b92..fe18994b83899 100644 --- a/flink-core/src/main/java/org/apache/flink/api/common/functions/util/AbstractRuntimeUDFContext.java +++ b/flink-core/src/main/java/org/apache/flink/api/common/functions/util/AbstractRuntimeUDFContext.java @@ -34,7 +34,9 @@ import org.apache.flink.api.common.accumulators.LongCounter; import org.apache.flink.api.common.cache.DistributedCache; import org.apache.flink.api.common.functions.RuntimeContext; -import org.apache.flink.api.common.state.OperatorState; +import org.apache.flink.api.common.state.ValueState; +import org.apache.flink.api.common.state.State; +import org.apache.flink.api.common.state.StateDescriptor; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.core.fs.Path; @@ -169,13 +171,22 @@ private Accumulator getAccumulator(String name } @Override - public OperatorState getKeyValueState(String name, Class stateType, S defaultState) { + public S getPartitionedState(StateDescriptor stateDescriptor) { + throw new UnsupportedOperationException( + "This state is only accessible by functions executed on a KeyedStream"); + + } + + @Override + @Deprecated + public ValueState getKeyValueState(String name, Class stateType, S defaultState) { throw new UnsupportedOperationException( "This state is only accessible by functions executed on a KeyedStream"); } @Override - public OperatorState getKeyValueState(String name, TypeInformation stateType, S defaultState) { + @Deprecated + public ValueState getKeyValueState(String name, TypeInformation stateType, S defaultState) { throw new UnsupportedOperationException( "This state is only accessible by functions executed on a KeyedStream"); } diff --git a/flink-core/src/main/java/org/apache/flink/api/common/state/ListState.java b/flink-core/src/main/java/org/apache/flink/api/common/state/ListState.java new file mode 100644 index 0000000000000..f803105982445 --- /dev/null +++ b/flink-core/src/main/java/org/apache/flink/api/common/state/ListState.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.api.common.state; + +/** + * {@link State} interface for partitioned list state in Operations. + * The state is accessed and modified by user functions, and checkpointed consistently + * by the system as part of the distributed snapshots. + * + *

The state is only accessible by functions applied on a KeyedDataStream. The key is + * automatically supplied by the system, so the function always sees the value mapped to the + * key of the current element. That way, the system can handle stream and state partitioning + * consistently together. + * + * @param Type of values that this list state keeps. + */ +public interface ListState extends MergingState> {} diff --git a/flink-core/src/main/java/org/apache/flink/api/common/state/ListStateDescriptor.java b/flink-core/src/main/java/org/apache/flink/api/common/state/ListStateDescriptor.java new file mode 100644 index 0000000000000..e39112627bc16 --- /dev/null +++ b/flink-core/src/main/java/org/apache/flink/api/common/state/ListStateDescriptor.java @@ -0,0 +1,87 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.flink.api.common.state; + +import org.apache.flink.api.common.typeutils.TypeSerializer; + +import static java.util.Objects.requireNonNull; + +/** + * {@link StateDescriptor} for {@link ListState}. This can be used to create a partitioned + * list state using + * {@link org.apache.flink.api.common.functions.RuntimeContext#getPartitionedState(StateDescriptor)}. + * + * @param The type of the values that can be added to the list state. + */ +public class ListStateDescriptor extends StateDescriptor> { + private static final long serialVersionUID = 1L; + + private final TypeSerializer serializer; + + /** + * Creates a new {@code ListStateDescriptor} with the given name. + * + * @param name The (unique) name for the state. + * @param serializer {@link TypeSerializer} for the state values. + */ + public ListStateDescriptor(String name, TypeSerializer serializer) { + super(requireNonNull(name)); + this.serializer = requireNonNull(serializer); + } + + @Override + public ListState bind(StateBackend stateBackend) throws Exception { + return stateBackend.createListState(this); + } + + /** + * Returns the {@link TypeSerializer} that can be used to serialize the value in the state. + */ + public TypeSerializer getSerializer() { + return serializer; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + + ListStateDescriptor that = (ListStateDescriptor) o; + + return serializer.equals(that.serializer) && name.equals(that.name); + + } + + @Override + public int hashCode() { + int result = serializer.hashCode(); + result = 31 * result + name.hashCode(); + return result; + } + + @Override + public String toString() { + return "ListStateDescriptor{" + + "serializer=" + serializer + + '}'; + } +} diff --git a/flink-core/src/main/java/org/apache/flink/api/common/state/MergingState.java b/flink-core/src/main/java/org/apache/flink/api/common/state/MergingState.java new file mode 100644 index 0000000000000..f6c0ecb547c8e --- /dev/null +++ b/flink-core/src/main/java/org/apache/flink/api/common/state/MergingState.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.api.common.state; + +import java.io.IOException; + +/** + * Base interface for partitioned state that supports adding elements and inspecting the current + * state of merged elements. Elements can either be kept in a buffer (list-like) or merged together + * into one value. + * + *

The state is accessed and modified by user functions, and checkpointed consistently + * by the system as part of the distributed snapshots. + * + *

The state is only accessible by functions applied on a KeyedDataStream. The key is + * automatically supplied by the system, so the function always sees the value mapped to the + * key of the current element. That way, the system can handle stream and state partitioning + * consistently together. + * + * @param Type of the value that can be added to the state. + * @param Type of the value that can be retrieved from the state. + */ +public interface MergingState extends State { + + /** + * Returns the current value for the state. When the state is not + * partitioned the returned value is the same for all inputs in a given + * operator instance. If state partitioning is applied, the value returned + * depends on the current operator input, as the operator maintains an + * independent state for each partition. + * + * @return The operator state value corresponding to the current input. + * + * @throws Exception Thrown if the system cannot access the state. + */ + OUT get() throws Exception ; + + /** + * Updates the operator state accessible by {@link #get()} by adding the given value + * to the list of values. The next time {@link #get()} is called (for the same state + * partition) the returned state will represent the updated list. + * + * @param value + * The new value for the state. + * + * @throws IOException Thrown if the system cannot access the state. + */ + void add(IN value) throws Exception; + +} diff --git a/flink-core/src/main/java/org/apache/flink/api/common/state/OperatorState.java b/flink-core/src/main/java/org/apache/flink/api/common/state/OperatorState.java index ec30f82d27a1a..32ffa7ff6d778 100644 --- a/flink-core/src/main/java/org/apache/flink/api/common/state/OperatorState.java +++ b/flink-core/src/main/java/org/apache/flink/api/common/state/OperatorState.java @@ -35,6 +35,7 @@ * @param Type of the value in the operator state */ @Public +@Deprecated public interface OperatorState { /** diff --git a/flink-core/src/main/java/org/apache/flink/api/common/state/ReducingState.java b/flink-core/src/main/java/org/apache/flink/api/common/state/ReducingState.java new file mode 100644 index 0000000000000..3e2c543553186 --- /dev/null +++ b/flink-core/src/main/java/org/apache/flink/api/common/state/ReducingState.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.api.common.state; + +/** + * {@link State} interface for reducing state. Elements can be added to the state, they will + * be combined using a reduce function. The current state can be inspected. + * + *

The state is accessed and modified by user functions, and checkpointed consistently + * by the system as part of the distributed snapshots. + * + *

The state is only accessible by functions applied on a KeyedDataStream. The key is + * automatically supplied by the system, so the function always sees the value mapped to the + * key of the current element. That way, the system can handle stream and state partitioning + * consistently together. + * + * @param Type of the value in the operator state + */ +public interface ReducingState extends MergingState {} diff --git a/flink-core/src/main/java/org/apache/flink/api/common/state/ReducingStateDescriptor.java b/flink-core/src/main/java/org/apache/flink/api/common/state/ReducingStateDescriptor.java new file mode 100644 index 0000000000000..7153a0555dd87 --- /dev/null +++ b/flink-core/src/main/java/org/apache/flink/api/common/state/ReducingStateDescriptor.java @@ -0,0 +1,106 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.api.common.state; + +import org.apache.flink.api.common.functions.ReduceFunction; +import org.apache.flink.api.common.functions.RichFunction; +import org.apache.flink.api.common.typeutils.TypeSerializer; + +import static java.util.Objects.requireNonNull; + +/** + * {@link StateDescriptor} for {@link ReducingState}. This can be used to create partitioned + * reducing state using + * {@link org.apache.flink.api.common.functions.RuntimeContext#getPartitionedState(StateDescriptor)}. + * + * @param The type of the values that can be added to the list state. + */ +public class ReducingStateDescriptor extends StateDescriptor> { + private static final long serialVersionUID = 1L; + + private final TypeSerializer serializer; + + private final ReduceFunction reduceFunction; + + /** + * Creates a new {@code ReducingStateDescriptor} with the given name and reduce function. + * + * @param name The (unique) name for the state. + * @param serializer {@link TypeSerializer} for the state values. + */ + public ReducingStateDescriptor(String name, + ReduceFunction reduceFunction, + TypeSerializer serializer) { + super(requireNonNull(name)); + if (reduceFunction instanceof RichFunction) { + throw new UnsupportedOperationException("ReduceFunction of ReducingState can not be a RichFunction."); + } + this.serializer = requireNonNull(serializer); + this.reduceFunction = reduceFunction; + } + + @Override + public ReducingState bind(StateBackend stateBackend) throws Exception { + return stateBackend.createReducingState(this); + } + + /** + * Returns the {@link TypeSerializer} that can be used to serialize the value in the state. + */ + public TypeSerializer getSerializer() { + return serializer; + } + + /** + * Returns the reduce function to be used for the reducing state. + */ + public ReduceFunction getReduceFunction() { + return reduceFunction; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + + ReducingStateDescriptor that = (ReducingStateDescriptor) o; + + return serializer.equals(that.serializer) && name.equals(that.name); + + } + + @Override + public int hashCode() { + int result = serializer.hashCode(); + result = 31 * result + name.hashCode(); + return result; + } + + @Override + public String toString() { + return "ReducingStateDescriptor{" + + "serializer=" + serializer + + ", reduceFunction=" + reduceFunction + + '}'; + } +} diff --git a/flink-core/src/main/java/org/apache/flink/api/common/state/State.java b/flink-core/src/main/java/org/apache/flink/api/common/state/State.java new file mode 100644 index 0000000000000..255a735dcf9d2 --- /dev/null +++ b/flink-core/src/main/java/org/apache/flink/api/common/state/State.java @@ -0,0 +1,30 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.flink.api.common.state; + +/** + * Interface that different types of partitioned state must implement. + * + *

The state is only accessible by functions applied on a KeyedDataStream. The key is + * automatically supplied by the system, so the function always sees the value mapped to the + * key of the current element. That way, the system can handle stream and state partitioning + * consistently together. + */ +public interface State { + void clear(); +} diff --git a/flink-core/src/main/java/org/apache/flink/api/common/state/StateBackend.java b/flink-core/src/main/java/org/apache/flink/api/common/state/StateBackend.java new file mode 100644 index 0000000000000..d5adf9becef75 --- /dev/null +++ b/flink-core/src/main/java/org/apache/flink/api/common/state/StateBackend.java @@ -0,0 +1,50 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.flink.api.common.state; + +/** + * The {@code StateBackend} is used by {@link StateDescriptor} instances to create actual state + * representations. + */ +public interface StateBackend { + + /** + * Creates and returns a new {@link ValueState}. + * @param stateDesc The {@code StateDescriptor} that contains the name of the state. + * + * @param The type of the value that the {@code ValueState} can store. + */ + ValueState createValueState(ValueStateDescriptor stateDesc) throws Exception; + + /** + * Creates and returns a new {@link ListState}. + * @param stateDesc The {@code StateDescriptor} that contains the name of the state. + * + * @param The type of the values that the {@code ListState} can store. + */ + ListState createListState(ListStateDescriptor stateDesc) throws Exception; + + /** + * Creates and returns a new {@link ReducingState}. + * @param stateDesc The {@code StateDescriptor} that contains the name of the state. + * + * @param The type of the values that the {@code ListState} can store. + */ + ReducingState createReducingState(ReducingStateDescriptor stateDesc) throws Exception; + +} diff --git a/flink-core/src/main/java/org/apache/flink/api/common/state/StateDescriptor.java b/flink-core/src/main/java/org/apache/flink/api/common/state/StateDescriptor.java new file mode 100644 index 0000000000000..f62118dd06755 --- /dev/null +++ b/flink-core/src/main/java/org/apache/flink/api/common/state/StateDescriptor.java @@ -0,0 +1,66 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.flink.api.common.state; + +import java.io.Serializable; + +import static java.util.Objects.requireNonNull; + +/** + * Base class for state descriptors. A {@code StateDescriptor} is used for creating partitioned + * {@link State} in stateful operations. This contains the name and can create an actual state + * object given a {@link StateBackend} using {@link #bind(StateBackend)}. + * + *

Subclasses must correctly implement {@link #equals(Object)} and {@link #hashCode()}. + * + * @param The type of the State objects created from this {@code StateDescriptor}. + */ +public abstract class StateDescriptor implements Serializable { + private static final long serialVersionUID = 1L; + + /** Name that uniquely identifies state created from this StateDescriptor. */ + protected final String name; + + /** + * Create a new {@code StateDescriptor} with the given name. + * @param name The name of the {@code StateDescriptor}. + */ + public StateDescriptor(String name) { + this.name = requireNonNull(name);; + } + + /** + * Returns the name of this {@code StateDescriptor}. + */ + public String getName() { + return name; + } + + /** + * Creates a new {@link State} on the given {@link StateBackend}. + * + * @param stateBackend The {@code StateBackend} on which to create the {@link State}. + */ + public abstract S bind(StateBackend stateBackend) throws Exception ; + + // Force subclasses to implement + public abstract boolean equals(Object o); + + // Force subclasses to implement + public abstract int hashCode(); +} diff --git a/flink-core/src/main/java/org/apache/flink/api/common/state/ValueState.java b/flink-core/src/main/java/org/apache/flink/api/common/state/ValueState.java new file mode 100644 index 0000000000000..ddb048f2e554a --- /dev/null +++ b/flink-core/src/main/java/org/apache/flink/api/common/state/ValueState.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.api.common.state; + +import org.apache.flink.annotation.Public; + +import java.io.IOException; + +/** + * {@link State} interface for partitioned single-value state. The value can be retrieved or + * updated. + * + *

The state is accessed and modified by user functions, and checkpointed consistently + * by the system as part of the distributed snapshots. + * + *

The state is only accessible by functions applied on a KeyedDataStream. The key is + * automatically supplied by the system, so the function always sees the value mapped to the + * key of the current element. That way, the system can handle stream and state partitioning + * consistently together. + * + * @param Type of the value in the state. + */ +@Public +public interface ValueState extends State, OperatorState { + + /** + * Returns the current value for the state. When the state is not + * partitioned the returned value is the same for all inputs in a given + * operator instance. If state partitioning is applied, the value returned + * depends on the current operator input, as the operator maintains an + * independent state for each partition. + * + * @return The operator state value corresponding to the current input. + * + * @throws IOException Thrown if the system cannot access the state. + */ + T value() throws IOException; + + /** + * Updates the operator state accessible by {@link #value()} to the given + * value. The next time {@link #value()} is called (for the same state + * partition) the returned state will represent the updated value. When a + * partitioned state is updated with null, the state for the current key + * will be removed and the default value is returned on the next access. + * + * @param value + * The new value for the state. + * + * @throws IOException Thrown if the system cannot access the state. + */ + void update(T value) throws IOException; + +} diff --git a/flink-core/src/main/java/org/apache/flink/api/common/state/ValueStateDescriptor.java b/flink-core/src/main/java/org/apache/flink/api/common/state/ValueStateDescriptor.java new file mode 100644 index 0000000000000..bcfa46f5328b7 --- /dev/null +++ b/flink-core/src/main/java/org/apache/flink/api/common/state/ValueStateDescriptor.java @@ -0,0 +1,166 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.flink.api.common.state; + +import org.apache.flink.api.common.typeutils.TypeSerializer; +import org.apache.flink.core.memory.DataInputViewStreamWrapper; +import org.apache.flink.core.memory.DataOutputViewStreamWrapper; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; + +import static java.util.Objects.requireNonNull; + +/** + * {@link StateDescriptor} for {@link ValueState}. This can be used to create partitioned + * value state using + * {@link org.apache.flink.api.common.functions.RuntimeContext#getPartitionedState(StateDescriptor)}. + * + * @param The type of the values that the value state can hold. + */ +public class ValueStateDescriptor extends StateDescriptor> { + private static final long serialVersionUID = 1L; + + private transient T defaultValue; + + private final TypeSerializer serializer; + + /** + * Creates a new {@code ValueStateDescriptor} with the given name and default value. + * + * @param name The (unique) name for the state. + * @param defaultValue The default value that will be set when requesting state without setting + * a value before. + * @param serializer {@link TypeSerializer} for the state values. + */ + public ValueStateDescriptor(String name, T defaultValue, TypeSerializer serializer) { + super(requireNonNull(name)); + this.defaultValue = defaultValue; + this.serializer = requireNonNull(serializer); + } + + private void writeObject(final ObjectOutputStream out) throws IOException { + out.defaultWriteObject(); + + if (defaultValue == null) { + // we don't have a default value + out.writeBoolean(false); + } else { + out.writeBoolean(true); + + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + DataOutputViewStreamWrapper outView = + new DataOutputViewStreamWrapper(new DataOutputStream(baos)); + + try { + serializer.serialize(defaultValue, outView); + } catch (IOException ioe) { + throw new RuntimeException("Unable to serialize default value of type " + + defaultValue.getClass().getSimpleName() + ".", ioe); + } + + outView.close(); + + out.writeInt(baos.size()); + out.write(baos.toByteArray()); + } + + } + + private void readObject(final ObjectInputStream in) throws IOException, ClassNotFoundException { + in.defaultReadObject(); + + boolean hasDefaultValue = in.readBoolean(); + + if (hasDefaultValue) { + int size = in.readInt(); + byte[] buffer = new byte[size]; + int bytesRead = in.read(buffer); + + if (bytesRead != size) { + throw new RuntimeException("Read size does not match expected size."); + } + + ByteArrayInputStream bais = new ByteArrayInputStream(buffer); + DataInputViewStreamWrapper inView = + new DataInputViewStreamWrapper(new DataInputStream(bais)); + defaultValue = serializer.deserialize(inView); + } else { + defaultValue = null; + } + } + + @Override + public ValueState bind(StateBackend stateBackend) throws Exception { + return stateBackend.createValueState(this); + } + + /** + * Returns the default value. + */ + public T getDefaultValue() { + if (defaultValue != null) { + return serializer.copy(defaultValue); + } else { + return null; + } + } + + /** + * Returns the {@link TypeSerializer} that can be used to serialize the value in the state. + */ + public TypeSerializer getSerializer() { + return serializer; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + + ValueStateDescriptor that = (ValueStateDescriptor) o; + + return serializer.equals(that.serializer) && name.equals(that.name); + + } + + @Override + public int hashCode() { + int result = serializer.hashCode(); + result = 31 * result + name.hashCode(); + return result; + } + + @Override + public String toString() { + return "ValueStateDescriptor{" + + "name=" + name + + ", defaultValue=" + defaultValue + + ", serializer=" + serializer + + '}'; + } +} diff --git a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/TypeSerializerSingleton.java b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/TypeSerializerSingleton.java index 68842d697bcac..a9d5cd6f199a4 100644 --- a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/TypeSerializerSingleton.java +++ b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/TypeSerializerSingleton.java @@ -33,7 +33,7 @@ public TypeSerializerSingleton duplicate() { @Override public int hashCode() { - return TypeSerializerSingleton.class.hashCode(); + return this.getClass().hashCode(); } @Override diff --git a/flink-examples/flink-examples-streaming/src/main/java/org/apache/flink/streaming/examples/windowing/SessionWindowing.java b/flink-examples/flink-examples-streaming/src/main/java/org/apache/flink/streaming/examples/windowing/SessionWindowing.java index 035727a2c90de..7938ee408a9bd 100644 --- a/flink-examples/flink-examples-streaming/src/main/java/org/apache/flink/streaming/examples/windowing/SessionWindowing.java +++ b/flink-examples/flink-examples-streaming/src/main/java/org/apache/flink/streaming/examples/windowing/SessionWindowing.java @@ -17,7 +17,7 @@ package org.apache.flink.streaming.examples.windowing; -import org.apache.flink.api.common.state.OperatorState; +import org.apache.flink.api.common.state.ValueState; import org.apache.flink.api.java.tuple.Tuple3; import org.apache.flink.streaming.api.TimeCharacteristic; import org.apache.flink.streaming.api.datastream.DataStream; @@ -108,7 +108,7 @@ public SessionTrigger(Long sessionTimeout) { @Override public TriggerResult onElement(Tuple3 element, long timestamp, GlobalWindow window, TriggerContext ctx) throws Exception { - OperatorState lastSeenState = ctx.getKeyValueState("last-seen", 1L); + ValueState lastSeenState = ctx.getKeyValueState("last-seen", 1L); Long lastSeen = lastSeenState.value(); Long timeSinceLastEvent = timestamp - lastSeen; @@ -127,7 +127,7 @@ public TriggerResult onElement(Tuple3 element, long times @Override public TriggerResult onEventTime(long time, GlobalWindow window, TriggerContext ctx) throws Exception { - OperatorState lastSeenState = ctx.getKeyValueState("last-seen", 1L); + ValueState lastSeenState = ctx.getKeyValueState("last-seen", 1L); Long lastSeen = lastSeenState.value(); if (time - lastSeen >= sessionTimeout) { diff --git a/flink-fs-tests/src/test/java/org/apache/flink/hdfstests/FileStateBackendTest.java b/flink-fs-tests/src/test/java/org/apache/flink/hdfstests/FileStateBackendTest.java index 49dfc21b45f4b..942a3c9934f1b 100644 --- a/flink-fs-tests/src/test/java/org/apache/flink/hdfstests/FileStateBackendTest.java +++ b/flink-fs-tests/src/test/java/org/apache/flink/hdfstests/FileStateBackendTest.java @@ -25,11 +25,14 @@ import org.apache.flink.core.fs.FileSystem; import org.apache.flink.core.fs.Path; import org.apache.flink.core.testutils.CommonTestUtils; +import org.apache.flink.runtime.state.StateBackendTestBase; import org.apache.flink.runtime.state.StateHandle; import org.apache.flink.runtime.state.filesystem.FileStreamStateHandle; import org.apache.flink.runtime.state.filesystem.FsStateBackend; import org.apache.flink.runtime.operators.testutils.DummyEnvironment; -import org.apache.flink.runtime.state.StateBackend; +import org.apache.flink.api.common.typeutils.base.IntSerializer; + +import org.apache.flink.runtime.state.AbstractStateBackend; import org.apache.flink.runtime.state.StreamStateHandle; import org.apache.flink.runtime.state.memory.ByteStreamStateHandle; import org.apache.hadoop.conf.Configuration; @@ -55,7 +58,7 @@ import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; -public class FileStateBackendTest { +public class FileStateBackendTest extends StateBackendTestBase { private static File TEMP_DIR; @@ -99,6 +102,20 @@ public static void destroyHDFS() { catch (Exception ignored) {} } + private URI stateBaseURI; + + @Override + protected FsStateBackend getStateBackend() throws Exception { + stateBaseURI = new URI(HDFS_ROOT_URI + UUID.randomUUID().toString()); + return new FsStateBackend(stateBaseURI); + + } + + @Override + protected void cleanup() throws Exception { + FileSystem.get(stateBaseURI).delete(new Path(stateBaseURI), true); + } + // ------------------------------------------------------------------------ // Tests // ------------------------------------------------------------------------ @@ -128,7 +145,7 @@ public void testSetupAndSerialization() { // supreme! } - backend.initializeForJob(new DummyEnvironment("test", 1, 0)); + backend.initializeForJob(new DummyEnvironment("test", 1, 0), "dummy", IntSerializer.INSTANCE); assertNotNull(backend.getCheckpointDirectory()); Path checkpointDir = backend.getCheckpointDirectory(); @@ -149,9 +166,8 @@ public void testSetupAndSerialization() { @Test public void testSerializableState() { try { - FsStateBackend backend = CommonTestUtils.createCopySerializable( - new FsStateBackend(randomHdfsFileUri(), 40)); - backend.initializeForJob(new DummyEnvironment("test", 1, 0)); + FsStateBackend backend = CommonTestUtils.createCopySerializable(new FsStateBackend(randomHdfsFileUri(), 40)); + backend.initializeForJob(new DummyEnvironment("test", 1, 0), "dummy", IntSerializer.INSTANCE); Path checkpointDir = backend.getCheckpointDirectory(); @@ -183,9 +199,8 @@ public void testSerializableState() { @Test public void testStateOutputStream() { try { - FsStateBackend backend = CommonTestUtils.createCopySerializable( - new FsStateBackend(randomHdfsFileUri(), 15)); - backend.initializeForJob(new DummyEnvironment("test", 1, 0)); + FsStateBackend backend = CommonTestUtils.createCopySerializable(new FsStateBackend(randomHdfsFileUri(), 15)); + backend.initializeForJob(new DummyEnvironment("test", 1, 0), "dummy", IntSerializer.INSTANCE); Path checkpointDir = backend.getCheckpointDirectory(); @@ -219,14 +234,14 @@ public void testStateOutputStream() { // use with try-with-resources StreamStateHandle handle4; - try (StateBackend.CheckpointStateOutputStream stream4 = + try (AbstractStateBackend.CheckpointStateOutputStream stream4 = backend.createCheckpointStateOutputStream(checkpointId, System.currentTimeMillis())) { stream4.write(state4); handle4 = stream4.closeAndGetHandle(); } // close before accessing handle - StateBackend.CheckpointStateOutputStream stream5 = + AbstractStateBackend.CheckpointStateOutputStream stream5 = backend.createCheckpointStateOutputStream(checkpointId, System.currentTimeMillis()); stream5.write(state4); stream5.close(); diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/AbstractHeapKvState.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/AbstractHeapKvState.java deleted file mode 100644 index 23703b3fe9415..0000000000000 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/state/AbstractHeapKvState.java +++ /dev/null @@ -1,146 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.runtime.state; - -import org.apache.flink.api.common.typeutils.TypeSerializer; -import org.apache.flink.core.memory.DataOutputView; - -import java.io.IOException; -import java.util.HashMap; -import java.util.Map; - -import static java.util.Objects.requireNonNull; - -/** - * Base class for key/value state implementations that are backed by a regular heap hash map. The - * concrete implementations define how the state is checkpointed. - * - * @param The type of the key. - * @param The type of the value. - * @param The type of the backend that snapshots this key/value state. - */ -public abstract class AbstractHeapKvState> implements KvState { - - /** Map containing the actual key/value pairs */ - private final HashMap state; - - /** The serializer for the keys */ - private final TypeSerializer keySerializer; - - /** The serializer for the values */ - private final TypeSerializer valueSerializer; - - /** The value that is returned when no other value has been associated with a key, yet */ - private final V defaultValue; - - /** The current key, which the next value methods will refer to */ - private K currentKey; - - /** - * Creates a new empty key/value state. - * - * @param keySerializer The serializer for the keys. - * @param valueSerializer The serializer for the values. - * @param defaultValue The value that is returned when no other value has been associated with a key, yet. - */ - protected AbstractHeapKvState(TypeSerializer keySerializer, - TypeSerializer valueSerializer, - V defaultValue) { - this(keySerializer, valueSerializer, defaultValue, new HashMap()); - } - - /** - * Creates a new key/value state for the given hash map of key/value pairs. - * - * @param keySerializer The serializer for the keys. - * @param valueSerializer The serializer for the values. - * @param defaultValue The value that is returned when no other value has been associated with a key, yet. - * @param state The state map to use in this kev/value state. May contain initial state. - */ - protected AbstractHeapKvState(TypeSerializer keySerializer, - TypeSerializer valueSerializer, - V defaultValue, - HashMap state) { - this.state = requireNonNull(state); - this.keySerializer = requireNonNull(keySerializer); - this.valueSerializer = requireNonNull(valueSerializer); - this.defaultValue = defaultValue; - } - - // ------------------------------------------------------------------------ - - @Override - public V value() { - V value = state.get(currentKey); - return value != null ? value : - (defaultValue == null ? null : valueSerializer.copy(defaultValue)); - } - - @Override - public void update(V value) { - if (value != null) { - state.put(currentKey, value); - } - else { - state.remove(currentKey); - } - } - - @Override - public void setCurrentKey(K currentKey) { - this.currentKey = currentKey; - } - - @Override - public int size() { - return state.size(); - } - - @Override - public void dispose() { - state.clear(); - } - - /** - * Gets the serializer for the keys. - * @return The serializer for the keys. - */ - public TypeSerializer getKeySerializer() { - return keySerializer; - } - - /** - * Gets the serializer for the values. - * @return The serializer for the values. - */ - public TypeSerializer getValueSerializer() { - return valueSerializer; - } - - // ------------------------------------------------------------------------ - // checkpointing utilities - // ------------------------------------------------------------------------ - - protected void writeStateToOutputView(final DataOutputView out) throws IOException { - for (Map.Entry entry : state.entrySet()) { - keySerializer.serialize(entry.getKey(), out); - valueSerializer.serialize(entry.getValue(), out); - } - } -} diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/AbstractHeapState.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/AbstractHeapState.java new file mode 100644 index 0000000000000..206be6422763a --- /dev/null +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/AbstractHeapState.java @@ -0,0 +1,164 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.runtime.state; + +import org.apache.flink.api.common.state.ListState; +import org.apache.flink.api.common.state.State; +import org.apache.flink.api.common.state.StateDescriptor; +import org.apache.flink.api.common.typeutils.TypeSerializer; + +import java.util.HashMap; +import java.util.Map; + +import static java.util.Objects.requireNonNull; + +/** + * Base class for partitioned {@link ListState} implementations that are backed by a regular + * heap hash map. The concrete implementations define how the state is checkpointed. + * + * @param The type of the key. + * @param The type of the namespace. + * @param The type of the values in the state. + * @param The type of State + * @param The type of StateDescriptor for the State S + * @param The type of the backend that snapshots this key/value state. + */ +public abstract class AbstractHeapState, Backend extends AbstractStateBackend> + implements KvState, State { + + /** Map containing the actual key/value pairs */ + protected final HashMap> state; + + /** Serializer for the state value. The state value could be a List, for example. */ + protected final TypeSerializer stateSerializer; + + /** The serializer for the keys */ + protected final TypeSerializer keySerializer; + + /** The serializer for the namespace */ + protected final TypeSerializer namespaceSerializer; + + /** This holds the name of the state and can create an initial default value for the state. */ + protected final SD stateDesc; + + /** The current key, which the next value methods will refer to */ + protected K currentKey; + + /** The current namespace, which the access methods will refer to. */ + protected N currentNamespace = null; + + /** Cache the state map for the current key. */ + protected Map currentNSState; + + /** + * Creates a new empty key/value state. + * + * @param keySerializer The serializer for the keys. + * @param namespaceSerializer The serializer for the namespace. + * @param stateDesc The state identifier for the state. This contains name + * and can create a default state value. + */ + protected AbstractHeapState(TypeSerializer keySerializer, + TypeSerializer namespaceSerializer, + TypeSerializer stateSerializer, + SD stateDesc) { + this(keySerializer, namespaceSerializer, stateSerializer, stateDesc, new HashMap>()); + } + + /** + * Creates a new key/value state for the given hash map of key/value pairs. + * + * @param keySerializer The serializer for the keys. + * @param stateDesc The state identifier for the state. This contains name + * and can create a default state value. + * @param state The state map to use in this kev/value state. May contain initial state. + */ + protected AbstractHeapState(TypeSerializer keySerializer, + TypeSerializer namespaceSerializer, + TypeSerializer stateSerializer, + SD stateDesc, + HashMap> state) { + this.state = requireNonNull(state); + this.keySerializer = requireNonNull(keySerializer); + this.namespaceSerializer = requireNonNull(namespaceSerializer); + this.stateSerializer = stateSerializer; + this.stateDesc = stateDesc; + } + + // ------------------------------------------------------------------------ + + @Override + public final void clear() { + if (currentNSState != null) { + currentNSState.remove(currentKey); + if (currentNSState.isEmpty()) { + state.remove(currentNamespace); + currentNSState = null; + } + } + } + + @Override + public final void setCurrentKey(K currentKey) { + this.currentKey = currentKey; + } + + @Override + public final void setCurrentNamespace(N namespace) { + if (namespace != null && namespace.equals(this.currentNamespace)) { + return; + } + this.currentNamespace = namespace; + this.currentNSState = state.get(currentNamespace); + } + + /** + * Returns the number of all state pairs in this state, across namespaces. + */ + protected final int size() { + int size = 0; + for (Map namespace: state.values()) { + size += namespace.size(); + } + return size; + } + + @Override + public void dispose() { + state.clear(); + } + + /** + * Gets the serializer for the keys. + * + * @return The serializer for the keys. + */ + public final TypeSerializer getKeySerializer() { + return keySerializer; + } + + /** + * Gets the serializer for the namespace. + * + * @return The serializer for the namespace. + */ + public final TypeSerializer getNamespaceSerializer() { + return namespaceSerializer; + } +} diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/AbstractStateBackend.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/AbstractStateBackend.java new file mode 100644 index 0000000000000..958b4dc639225 --- /dev/null +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/AbstractStateBackend.java @@ -0,0 +1,406 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.runtime.state; + +import org.apache.flink.api.common.state.ListState; +import org.apache.flink.api.common.state.ListStateDescriptor; +import org.apache.flink.api.common.state.ReducingState; +import org.apache.flink.api.common.state.ReducingStateDescriptor; +import org.apache.flink.api.common.state.State; +import org.apache.flink.api.common.state.StateBackend; +import org.apache.flink.api.common.state.StateDescriptor; +import org.apache.flink.api.common.state.ValueState; +import org.apache.flink.api.common.state.ValueStateDescriptor; +import org.apache.flink.api.common.typeutils.TypeSerializer; +import org.apache.flink.core.memory.DataInputView; +import org.apache.flink.core.memory.DataInputViewStreamWrapper; +import org.apache.flink.core.memory.DataOutputView; +import org.apache.flink.core.memory.DataOutputViewStreamWrapper; +import org.apache.flink.runtime.execution.Environment; + +import java.io.IOException; +import java.io.OutputStream; +import java.io.Serializable; +import java.util.HashMap; +import java.util.Map; + +/** + * A state backend defines how state is stored and snapshotted during checkpoints. + */ +public abstract class AbstractStateBackend implements java.io.Serializable { + + private static final long serialVersionUID = 4620413814639220247L; + + protected transient TypeSerializer keySerializer; + + protected transient ClassLoader userCodeClassLoader; + + protected transient Object currentKey; + + /** For efficient access in setCurrentKey() */ + private transient KvState[] keyValueStates; + + /** So that we can give out state when the user uses the same key. */ + private transient HashMap> keyValueStatesByName; + + /** For caching the last accessed partitioned state */ + private transient String lastName; + + @SuppressWarnings("rawtypes") + private transient KvState lastState; + + // ------------------------------------------------------------------------ + // initialization and cleanup + // ------------------------------------------------------------------------ + + /** + * This method is called by the task upon deployment to initialize the state backend for + * data for a specific job. + * + * @param env The {@link Environment} of the task that instantiated the state backend + * @param operatorIdentifier Unique identifier for naming states created by this backend + * @throws Exception Overwritten versions of this method may throw exceptions, in which + * case the job that uses the state backend is considered failed during + * deployment. + */ + public void initializeForJob(Environment env, + String operatorIdentifier, + TypeSerializer keySerializer) throws Exception { + this.userCodeClassLoader = env.getUserClassLoader(); + this.keySerializer = keySerializer; + } + + /** + * Disposes all state associated with the current job. + * + * @throws Exception Exceptions may occur during disposal of the state and should be forwarded. + */ + public abstract void disposeAllStateForCurrentJob() throws Exception; + + /** + * Closes the state backend, releasing all internal resources, but does not delete any persistent + * checkpoint data. + * + * @throws Exception Exceptions can be forwarded and will be logged by the system + */ + public abstract void close() throws Exception; + + public void dispose() { + if (keyValueStates != null) { + for (KvState state : keyValueStates) { + state.dispose(); + } + } + } + + // ------------------------------------------------------------------------ + // key/value state + // ------------------------------------------------------------------------ + + /** + * Creates and returns a new {@link ValueState}. + * + * @param namespaceSerializer TypeSerializer for the state namespace. + * @param stateDesc The {@code StateDescriptor} that contains the name of the state. + * + * @param The type of the namespace. + * @param The type of the value that the {@code ValueState} can store. + */ + abstract protected ValueState createValueState(TypeSerializer namespaceSerializer, ValueStateDescriptor stateDesc) throws Exception; + + /** + * Creates and returns a new {@link ListState}. + * + * @param namespaceSerializer TypeSerializer for the state namespace. + * @param stateDesc The {@code StateDescriptor} that contains the name of the state. + * + * @param The type of the namespace. + * @param The type of the values that the {@code ListState} can store. + */ + abstract protected ListState createListState(TypeSerializer namespaceSerializer, ListStateDescriptor stateDesc) throws Exception; + + /** + * Creates and returns a new {@link ReducingState}. + * + * @param namespaceSerializer TypeSerializer for the state namespace. + * @param stateDesc The {@code StateDescriptor} that contains the name of the state. + * + * @param The type of the namespace. + * @param The type of the values that the {@code ListState} can store. + */ + abstract protected ReducingState createReducingState(TypeSerializer namespaceSerializer, ReducingStateDescriptor stateDesc) throws Exception; + + /** + * Sets the current key that is used for partitioned state. + * @param currentKey The current key. + */ + @SuppressWarnings({"unchecked", "rawtypes"}) + public void setCurrentKey(Object currentKey) { + this.currentKey = currentKey; + if (keyValueStates != null) { + for (KvState kv : keyValueStates) { + kv.setCurrentKey(currentKey); + } + } + } + + public Object getCurrentKey() { + return currentKey; + } + + /** + * Creates or retrieves a partitioned state backed by this state backend. + * + * @param stateDescriptor The state identifier for the state. This contains name + * and can create a default state value. + * @param The type of the key. + * @param The type of the namespace. + * @param The type of the state. + * + * @return A new key/value state backed by this backend. + * + * @throws Exception Exceptions may occur during initialization of the state and should be forwarded. + */ + @SuppressWarnings({"rawtypes", "unchecked"}) + public S getPartitionedState(final N namespace, final TypeSerializer namespaceSerializer, final StateDescriptor stateDescriptor) throws Exception { + + if (keySerializer == null) { + throw new Exception("State key serializer has not been configured in the config. " + + "This operation cannot use partitioned state."); + } + + if (keyValueStatesByName == null) { + keyValueStatesByName = new HashMap<>(); + } + + if (lastName != null && lastName.equals(stateDescriptor.getName())) { + lastState.setCurrentNamespace(namespace); + return (S) lastState; + } + + KvState previous = keyValueStatesByName.get(stateDescriptor.getName()); + if (previous != null) { + lastState = previous; + lastState.setCurrentNamespace(namespace); + lastName = stateDescriptor.getName(); + return (S) previous; + } + + // create a new blank key/value state + S kvstate = stateDescriptor.bind(new StateBackend() { + @Override + public ValueState createValueState(ValueStateDescriptor stateDesc) throws Exception { + return AbstractStateBackend.this.createValueState(namespaceSerializer, stateDesc); + } + + @Override + public ListState createListState(ListStateDescriptor stateDesc) throws Exception { + return AbstractStateBackend.this.createListState(namespaceSerializer, stateDesc); + } + + @Override + public ReducingState createReducingState(ReducingStateDescriptor stateDesc) throws Exception { + return AbstractStateBackend.this.createReducingState(namespaceSerializer, stateDesc); + } + }); + + keyValueStatesByName.put(stateDescriptor.getName(), (KvState) kvstate); + keyValueStates = keyValueStatesByName.values().toArray(new KvState[keyValueStatesByName.size()]); + + lastName = stateDescriptor.getName(); + lastState = (KvState) kvstate; + + ((KvState) kvstate).setCurrentKey(currentKey); + ((KvState) kvstate).setCurrentNamespace(namespace); + + return kvstate; + } + + public HashMap> snapshotPartitionedState(long checkpointId, long timestamp) throws Exception { + if (keyValueStates != null) { + HashMap> snapshots = new HashMap<>(keyValueStatesByName.size()); + + for (Map.Entry> entry : keyValueStatesByName.entrySet()) { + KvStateSnapshot snapshot = entry.getValue().snapshot(checkpointId, timestamp); + snapshots.put(entry.getKey(), snapshot); + } + return snapshots; + } + + return null; + } + + public void notifyOfCompletedCheckpoint(long checkpointId) throws Exception { + // We check whether the KvStates require notifications + if (keyValueStates != null) { + for (KvState kvstate : keyValueStates) { + if (kvstate instanceof CheckpointListener) { + ((CheckpointListener) kvstate).notifyCheckpointComplete(checkpointId); + } + } + } + } + + /** + * Injects K/V state snapshots for lazy restore. + * @param keyValueStateSnapshots The Map of snapshots + */ + @SuppressWarnings("unchecked,rawtypes") + public final void injectKeyValueStateSnapshots(HashMap keyValueStateSnapshots, long recoveryTimestamp) throws Exception { + if (keyValueStateSnapshots != null) { + if (keyValueStatesByName == null) { + keyValueStatesByName = new HashMap<>(); + } + + for (Map.Entry state : keyValueStateSnapshots.entrySet()) { + KvState kvState = state.getValue().restoreState(this, + keySerializer, + userCodeClassLoader, + recoveryTimestamp); + keyValueStatesByName.put(state.getKey(), kvState); + } + keyValueStates = keyValueStatesByName.values().toArray(new KvState[keyValueStatesByName.size()]); + } + } + + // ------------------------------------------------------------------------ + // storing state for a checkpoint + // ------------------------------------------------------------------------ + + /** + * Creates an output stream that writes into the state of the given checkpoint. When the stream + * is closes, it returns a state handle that can retrieve the state back. + * + * @param checkpointID The ID of the checkpoint. + * @param timestamp The timestamp of the checkpoint. + * @return An output stream that writes state for the given checkpoint. + * + * @throws Exception Exceptions may occur while creating the stream and should be forwarded. + */ + public abstract CheckpointStateOutputStream createCheckpointStateOutputStream( + long checkpointID, long timestamp) throws Exception; + + /** + * Creates a {@link DataOutputView} stream that writes into the state of the given checkpoint. + * When the stream is closes, it returns a state handle that can retrieve the state back. + * + * @param checkpointID The ID of the checkpoint. + * @param timestamp The timestamp of the checkpoint. + * @return An DataOutputView stream that writes state for the given checkpoint. + * + * @throws Exception Exceptions may occur while creating the stream and should be forwarded. + */ + public CheckpointStateOutputView createCheckpointStateOutputView( + long checkpointID, long timestamp) throws Exception { + return new CheckpointStateOutputView(createCheckpointStateOutputStream(checkpointID, timestamp)); + } + + /** + * Writes the given state into the checkpoint, and returns a handle that can retrieve the state back. + * + * @param state The state to be checkpointed. + * @param checkpointID The ID of the checkpoint. + * @param timestamp The timestamp of the checkpoint. + * @param The type of the state. + * + * @return A state handle that can retrieve the checkpoined state. + * + * @throws Exception Exceptions may occur during serialization / storing the state and should be forwarded. + */ + public abstract StateHandle checkpointStateSerializable( + S state, long checkpointID, long timestamp) throws Exception; + + + // ------------------------------------------------------------------------ + // Checkpoint state output stream + // ------------------------------------------------------------------------ + + /** + * A dedicated output stream that produces a {@link StreamStateHandle} when closed. + */ + public static abstract class CheckpointStateOutputStream extends OutputStream { + + /** + * Closes the stream and gets a state handle that can create an input stream + * producing the data written to this stream. + * + * @return A state handle that can create an input stream producing the data written to this stream. + * @throws IOException Thrown, if the stream cannot be closed. + */ + public abstract StreamStateHandle closeAndGetHandle() throws IOException; + } + + /** + * A dedicated DataOutputView stream that produces a {@code StateHandle} when closed. + */ + public static final class CheckpointStateOutputView extends DataOutputViewStreamWrapper { + + private final CheckpointStateOutputStream out; + + public CheckpointStateOutputView(CheckpointStateOutputStream out) { + super(out); + this.out = out; + } + + /** + * Closes the stream and gets a state handle that can create a DataInputView. + * producing the data written to this stream. + * + * @return A state handle that can create an input stream producing the data written to this stream. + * @throws IOException Thrown, if the stream cannot be closed. + */ + public StateHandle closeAndGetHandle() throws IOException { + return new DataInputViewHandle(out.closeAndGetHandle()); + } + + @Override + public void close() throws IOException { + out.close(); + } + } + + /** + * Simple state handle that resolved a {@link DataInputView} from a StreamStateHandle. + */ + private static final class DataInputViewHandle implements StateHandle { + + private static final long serialVersionUID = 2891559813513532079L; + + private final StreamStateHandle stream; + + private DataInputViewHandle(StreamStateHandle stream) { + this.stream = stream; + } + + @Override + public DataInputView getState(ClassLoader userCodeClassLoader) throws Exception { + return new DataInputViewStreamWrapper(stream.getState(userCodeClassLoader)); + } + + @Override + public void discardState() throws Exception { + stream.discardState(); + } + + @Override + public long getStateSize() throws Exception { + return stream.getStateSize(); + } + } +} diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/ArrayListSerializer.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/ArrayListSerializer.java new file mode 100644 index 0000000000000..3bad8b02bd4ee --- /dev/null +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/ArrayListSerializer.java @@ -0,0 +1,125 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.flink.runtime.state; + +import org.apache.flink.api.common.typeutils.TypeSerializer; +import org.apache.flink.core.memory.DataInputView; +import org.apache.flink.core.memory.DataOutputView; + +import java.io.IOException; +import java.util.ArrayList; + +@SuppressWarnings("ForLoopReplaceableByForEach") +final public class ArrayListSerializer extends TypeSerializer> { + + private static final long serialVersionUID = 1119562170939152304L; + + private final TypeSerializer elementSerializer; + + public ArrayListSerializer(TypeSerializer elementSerializer) { + this.elementSerializer = elementSerializer; + } + + @Override + public boolean isImmutableType() { + return false; + } + + @Override + public TypeSerializer> duplicate() { + TypeSerializer duplicateElement = elementSerializer.duplicate(); + return duplicateElement == elementSerializer ? this : new ArrayListSerializer(duplicateElement); + } + + @Override + public ArrayList createInstance() { + return new ArrayList<>(); + } + + @Override + public ArrayList copy(ArrayList from) { + ArrayList newList = new ArrayList<>(from.size()); + for (int i = 0; i < from.size(); i++) { + newList.add(elementSerializer.copy(from.get(i))); + } + return newList; + } + + @Override + public ArrayList copy(ArrayList from, ArrayList reuse) { + return copy(from); + } + + @Override + public int getLength() { + return -1; // var length + } + + @Override + public void serialize(ArrayList list, DataOutputView target) throws IOException { + final int size = list.size(); + target.writeInt(size); + for (int i = 0; i < size; i++) { + elementSerializer.serialize(list.get(i), target); + } + } + + @Override + public ArrayList deserialize(DataInputView source) throws IOException { + final int size = source.readInt(); + final ArrayList list = new ArrayList<>(size); + for (int i = 0; i < size; i++) { + list.add(elementSerializer.deserialize(source)); + } + return list; + } + + @Override + public ArrayList deserialize(ArrayList reuse, DataInputView source) throws IOException { + return deserialize(source); + } + + @Override + public void copy(DataInputView source, DataOutputView target) throws IOException { + // copy number of elements + final int num = source.readInt(); + target.writeInt(num); + for (int i = 0; i < num; i++) { + elementSerializer.copy(source, target); + } + } + + // -------------------------------------------------------------------- + + @Override + public boolean equals(Object obj) { + return obj == this || + (obj != null && obj.getClass() == getClass() && + elementSerializer.equals(((ArrayListSerializer) obj).elementSerializer)); + } + + @Override + public boolean canEqual(Object obj) { + return true; + } + + @Override + public int hashCode() { + return elementSerializer.hashCode(); + } +} diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/checkpoint/CheckpointNotifier.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/CheckpointListener.java similarity index 93% rename from flink-streaming-java/src/main/java/org/apache/flink/streaming/api/checkpoint/CheckpointNotifier.java rename to flink-runtime/src/main/java/org/apache/flink/runtime/state/CheckpointListener.java index c2d218289ec2b..1f1880501ef88 100644 --- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/checkpoint/CheckpointNotifier.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/CheckpointListener.java @@ -15,14 +15,14 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.flink.streaming.api.checkpoint; +package org.apache.flink.runtime.state; /** * This interface must be implemented by functions/operations that want to receive * a commit notification once a checkpoint has been completely acknowledged by all * participants. */ -public interface CheckpointNotifier { +public interface CheckpointListener { /** * This method is called as a notification once a distributed checkpoint has been completed. diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/GenericListState.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/GenericListState.java new file mode 100644 index 0000000000000..c20962f160559 --- /dev/null +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/GenericListState.java @@ -0,0 +1,132 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.flink.runtime.state; + +import org.apache.flink.api.common.state.ListState; +import org.apache.flink.api.common.state.ListStateDescriptor; +import org.apache.flink.api.common.state.ValueState; +import org.apache.flink.api.common.state.ValueStateDescriptor; +import org.apache.flink.api.common.typeutils.TypeSerializer; + +import java.util.ArrayList; +import java.util.Collections; + +/** + * Generic implementation of {@link ListState} based on a wrapped {@link ValueState}. + * + * @param The type of the key. + * @param The type of the namespace. + * @param The type of the values stored in this {@code ListState}. + * @param The type of {@link AbstractStateBackend} that manages this {@code KvState}. + * @param Generic type that extends both the underlying {@code ValueState} and {@code KvState}. + */ +public class GenericListState> & KvState>, ValueStateDescriptor>, Backend>> + implements ListState, KvState, ListStateDescriptor, Backend> { + + private final W wrappedState; + + @SuppressWarnings("unchecked") + public GenericListState(ValueState> wrappedState) { + if (!(wrappedState instanceof KvState)) { + throw new IllegalArgumentException("Wrapped state must be a KvState."); + } + this.wrappedState = (W) wrappedState; + } + + @Override + public void setCurrentKey(K key) { + wrappedState.setCurrentKey(key); + } + + @Override + public void setCurrentNamespace(N namespace) { + wrappedState.setCurrentNamespace(namespace); + } + + @Override + public KvStateSnapshot, ListStateDescriptor, Backend> snapshot( + long checkpointId, + long timestamp) throws Exception { + KvStateSnapshot>, ValueStateDescriptor>, Backend> wrappedSnapshot = wrappedState.snapshot( + checkpointId, + timestamp); + return new Snapshot<>(wrappedSnapshot); + } + + @Override + public void dispose() { + wrappedState.dispose(); + } + + @Override + public Iterable get() throws Exception { + ArrayList result = wrappedState.value(); + if (result == null) { + return Collections.emptyList(); + } + return result; + } + + @Override + public void add(T value) throws Exception { + ArrayList currentValue = wrappedState.value(); + if (currentValue == null) { + currentValue = new ArrayList<>(); + currentValue.add(value); + wrappedState.update(currentValue); + } else { + currentValue.add(value); + wrappedState.update(currentValue); + } + } + + @Override + public void clear() { + wrappedState.clear(); + } + + private static class Snapshot implements KvStateSnapshot, ListStateDescriptor, Backend> { + private static final long serialVersionUID = 1L; + + private final KvStateSnapshot>, ValueStateDescriptor>, Backend> wrappedSnapshot; + + public Snapshot(KvStateSnapshot>, ValueStateDescriptor>, Backend> wrappedSnapshot) { + this.wrappedSnapshot = wrappedSnapshot; + } + + @Override + @SuppressWarnings("unchecked") + public KvState, ListStateDescriptor, Backend> restoreState( + Backend stateBackend, + TypeSerializer keySerializer, + ClassLoader classLoader, + long recoveryTimestamp) throws Exception { + return new GenericListState((ValueState) wrappedSnapshot.restoreState(stateBackend, keySerializer, classLoader, recoveryTimestamp)); + } + + @Override + public void discardState() throws Exception { + wrappedSnapshot.discardState(); + } + + @Override + public long getStateSize() throws Exception { + return wrappedSnapshot.getStateSize(); + } + } +} diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/GenericReducingState.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/GenericReducingState.java new file mode 100644 index 0000000000000..1181c666f0bd8 --- /dev/null +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/GenericReducingState.java @@ -0,0 +1,129 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.flink.runtime.state; + +import org.apache.flink.api.common.functions.ReduceFunction; +import org.apache.flink.api.common.state.ReducingState; +import org.apache.flink.api.common.state.ReducingStateDescriptor; +import org.apache.flink.api.common.state.ValueState; +import org.apache.flink.api.common.state.ValueStateDescriptor; +import org.apache.flink.api.common.typeutils.TypeSerializer; + +/** + * Generic implementation of {@link ReducingState} based on a wrapped {@link ValueState}. + * + * @param The type of the key. + * @param The type of the namespace. + * @param The type of the values stored in this {@code ReducingState}. + * @param The type of {@link AbstractStateBackend} that manages this {@code KvState}. + * @param Generic type that extends both the underlying {@code ValueState} and {@code KvState}. + */ +public class GenericReducingState & KvState, ValueStateDescriptor, Backend>> + implements ReducingState, KvState, ReducingStateDescriptor, Backend> { + + private final W wrappedState; + private final ReduceFunction reduceFunction; + + @SuppressWarnings("unchecked") + public GenericReducingState(ValueState wrappedState, ReduceFunction reduceFunction) { + if (!(wrappedState instanceof KvState)) { + throw new IllegalArgumentException("Wrapped state must be a KvState."); + } + this.wrappedState = (W) wrappedState; + this.reduceFunction = reduceFunction; + } + + @Override + public void setCurrentKey(K key) { + wrappedState.setCurrentKey(key); + } + + @Override + public void setCurrentNamespace(N namespace) { + wrappedState.setCurrentNamespace(namespace); + } + + @Override + public KvStateSnapshot, ReducingStateDescriptor, Backend> snapshot( + long checkpointId, + long timestamp) throws Exception { + KvStateSnapshot, ValueStateDescriptor, Backend> wrappedSnapshot = wrappedState.snapshot( + checkpointId, + timestamp); + return new Snapshot<>(wrappedSnapshot, reduceFunction); + } + + @Override + public void dispose() { + wrappedState.dispose(); + } + + @Override + public T get() throws Exception { + return wrappedState.value(); + } + + @Override + public void add(T value) throws Exception { + T currentValue = wrappedState.value(); + if (currentValue == null) { + wrappedState.update(value); + } else { + wrappedState.update(reduceFunction.reduce(currentValue, value)); + } + } + + @Override + public void clear() { + wrappedState.clear(); + } + + private static class Snapshot implements KvStateSnapshot, ReducingStateDescriptor, Backend> { + private static final long serialVersionUID = 1L; + + private final KvStateSnapshot, ValueStateDescriptor, Backend> wrappedSnapshot; + + private final ReduceFunction reduceFunction; + + public Snapshot(KvStateSnapshot, ValueStateDescriptor, Backend> wrappedSnapshot, + ReduceFunction reduceFunction) { + this.wrappedSnapshot = wrappedSnapshot; + this.reduceFunction = reduceFunction; + } + + @Override + @SuppressWarnings("unchecked") + public KvState, ReducingStateDescriptor, Backend> restoreState( + Backend stateBackend, + TypeSerializer keySerializer, + ClassLoader classLoader, + long recoveryTimestamp) throws Exception { + return new GenericReducingState((ValueState) wrappedSnapshot.restoreState(stateBackend, keySerializer, classLoader, recoveryTimestamp), reduceFunction); + } + + @Override + public void discardState() throws Exception { + wrappedSnapshot.discardState(); + } + + @Override + public long getStateSize() throws Exception { + return wrappedSnapshot.getStateSize(); + } + } +} diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/KvState.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/KvState.java index ef2c882a2ea76..7a97dc02ec4c3 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/state/KvState.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/KvState.java @@ -18,7 +18,8 @@ package org.apache.flink.runtime.state; -import org.apache.flink.api.common.state.OperatorState; +import org.apache.flink.api.common.state.State; +import org.apache.flink.api.common.state.StateDescriptor; /** * Key/Value state implementation for user-defined state. The state is backed by a state @@ -29,18 +30,27 @@ * metadata of what is considered part of the checkpoint. * * @param The type of the key. - * @param The type of the value. + * @param The type of the namespace. + * @param The type of {@link State} this {@code KvState} holds. + * @param The type of the {@link StateDescriptor} for state {@code S}. + * @param The type of {@link AbstractStateBackend} that manages this {@code KvState}. */ -public interface KvState> extends OperatorState { +public interface KvState, Backend extends AbstractStateBackend> { /** - * Sets the current key, which will be used to retrieve values for the next calls to - * {@link #value()} and {@link #update(Object)}. - * + * Sets the current key, which will be used when using the state access methods. + * * @param key The key. */ void setCurrentKey(K key); + /** + * Sets the current namespace, which will be used when using the state access methods. + * + * @param namespace The namespace. + */ + void setCurrentNamespace(N namespace); + /** * Creates a snapshot of this state. * @@ -51,16 +61,7 @@ public interface KvState> extends Op * @throws Exception Exceptions during snapshotting the state should be forwarded, so the system * can react to failed snapshots. */ - KvStateSnapshot snapshot(long checkpointId, long timestamp) throws Exception; - - /** - * Gets the number of key/value pairs currently stored in the state. Note that is a key - * has been associated with "null", the key is removed from the state an will not - * be counted here. - * - * @return The number of key/value pairs currently stored in the state. - */ - int size(); + KvStateSnapshot snapshot(long checkpointId, long timestamp) throws Exception; /** * Disposes the key/value state, releasing all occupied resources. diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/KvStateSnapshot.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/KvStateSnapshot.java index 682c093a81ae5..ce72135d68c76 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/state/KvStateSnapshot.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/KvStateSnapshot.java @@ -18,6 +18,8 @@ package org.apache.flink.runtime.state; +import org.apache.flink.api.common.state.State; +import org.apache.flink.api.common.state.StateDescriptor; import org.apache.flink.api.common.typeutils.TypeSerializer; /** @@ -32,10 +34,12 @@ * a file and this snapshot object contains a pointer to that file. * * @param The type of the key - * @param The type of the value + * @param The type of the namespace + * @param The type of the {@link State} + * @param The type of the {@link StateDescriptor} * @param The type of the backend that can restore the state from this snapshot. */ -public interface KvStateSnapshot> extends java.io.Serializable { +public interface KvStateSnapshot, Backend extends AbstractStateBackend> extends java.io.Serializable { /** * Loads the key/value state back from this snapshot. @@ -43,21 +47,18 @@ public interface KvStateSnapshot> ex * @param stateBackend The state backend that created this snapshot and can restore the key/value state * from this snapshot. * @param keySerializer The serializer for the keys. - * @param valueSerializer The serializer for the values. - * @param defaultValue The value that is returned when no other value has been associated with a key, yet. * @param classLoader The class loader for user-defined types. - * + * @param recoveryTimestamp The timestamp of the checkpoint we are recovering from. + * * @return An instance of the key/value state loaded from this snapshot. * * @throws Exception Exceptions can occur during the state loading and are forwarded. */ - KvState restoreState( - Backend stateBackend, - TypeSerializer keySerializer, - TypeSerializer valueSerializer, - V defaultValue, - ClassLoader classLoader, - long recoveryTimestamp) throws Exception; + KvState restoreState( + Backend stateBackend, + TypeSerializer keySerializer, + ClassLoader classLoader, + long recoveryTimestamp) throws Exception; /** * Discards the state snapshot, removing any resources occupied by it. diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/StateBackend.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/StateBackend.java deleted file mode 100644 index 2c431251c3ecf..0000000000000 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/state/StateBackend.java +++ /dev/null @@ -1,220 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.runtime.state; - -import org.apache.flink.api.common.typeutils.TypeSerializer; -import org.apache.flink.core.memory.DataInputView; -import org.apache.flink.core.memory.DataInputViewStreamWrapper; -import org.apache.flink.core.memory.DataOutputView; -import org.apache.flink.core.memory.DataOutputViewStreamWrapper; -import org.apache.flink.runtime.execution.Environment; - -import java.io.IOException; -import java.io.OutputStream; -import java.io.Serializable; - -/** - * A state backend defines how state is stored and snapshotted during checkpoints. - * - * @param The type of backend itself. This generic parameter is used to refer to the - * type of backend when creating state backed by this backend. - */ -public abstract class StateBackend> implements java.io.Serializable { - - private static final long serialVersionUID = 4620413814639220247L; - - // ------------------------------------------------------------------------ - // initialization and cleanup - // ------------------------------------------------------------------------ - - /** - * This method is called by the task upon deployment to initialize the state backend for - * data for a specific job. - * - * @param env The {@link Environment} of the task that instantiated the state backend - * @throws Exception Overwritten versions of this method may throw exceptions, in which - * case the job that uses the state backend is considered failed during - * deployment. - */ - public abstract void initializeForJob(Environment env) throws Exception; - - /** - * Disposes all state associated with the current job. - * - * @throws Exception Exceptions may occur during disposal of the state and should be forwarded. - */ - public abstract void disposeAllStateForCurrentJob() throws Exception; - - /** - * Closes the state backend, releasing all internal resources, but does not delete any persistent - * checkpoint data. - * - * @throws Exception Exceptions can be forwarded and will be logged by the system - */ - public abstract void close() throws Exception; - - // ------------------------------------------------------------------------ - // key/value state - // ------------------------------------------------------------------------ - - /** - * Creates a key/value state backed by this state backend. - * - * @param stateId Unique id that identifies the kv state in the streaming program. - * @param stateName Name of the created state - * @param keySerializer The serializer for the key. - * @param valueSerializer The serializer for the value. - * @param defaultValue The value that is returned when no other value has been associated with a key, yet. - * @param The type of the key. - * @param The type of the value. - * - * @return A new key/value state backed by this backend. - * - * @throws Exception Exceptions may occur during initialization of the state and should be forwarded. - */ - public abstract KvState createKvState(String stateId, String stateName, - TypeSerializer keySerializer, TypeSerializer valueSerializer, - V defaultValue) throws Exception; - - - // ------------------------------------------------------------------------ - // storing state for a checkpoint - // ------------------------------------------------------------------------ - - /** - * Creates an output stream that writes into the state of the given checkpoint. When the stream - * is closes, it returns a state handle that can retrieve the state back. - * - * @param checkpointID The ID of the checkpoint. - * @param timestamp The timestamp of the checkpoint. - * @return An output stream that writes state for the given checkpoint. - * - * @throws Exception Exceptions may occur while creating the stream and should be forwarded. - */ - public abstract CheckpointStateOutputStream createCheckpointStateOutputStream( - long checkpointID, long timestamp) throws Exception; - - /** - * Creates a {@link DataOutputView} stream that writes into the state of the given checkpoint. - * When the stream is closes, it returns a state handle that can retrieve the state back. - * - * @param checkpointID The ID of the checkpoint. - * @param timestamp The timestamp of the checkpoint. - * @return An DataOutputView stream that writes state for the given checkpoint. - * - * @throws Exception Exceptions may occur while creating the stream and should be forwarded. - */ - public CheckpointStateOutputView createCheckpointStateOutputView( - long checkpointID, long timestamp) throws Exception { - return new CheckpointStateOutputView(createCheckpointStateOutputStream(checkpointID, timestamp)); - } - - /** - * Writes the given state into the checkpoint, and returns a handle that can retrieve the state back. - * - * @param state The state to be checkpointed. - * @param checkpointID The ID of the checkpoint. - * @param timestamp The timestamp of the checkpoint. - * @param The type of the state. - * - * @return A state handle that can retrieve the checkpoined state. - * - * @throws Exception Exceptions may occur during serialization / storing the state and should be forwarded. - */ - public abstract StateHandle checkpointStateSerializable( - S state, long checkpointID, long timestamp) throws Exception; - - - // ------------------------------------------------------------------------ - // Checkpoint state output stream - // ------------------------------------------------------------------------ - - /** - * A dedicated output stream that produces a {@link StreamStateHandle} when closed. - */ - public static abstract class CheckpointStateOutputStream extends OutputStream { - - /** - * Closes the stream and gets a state handle that can create an input stream - * producing the data written to this stream. - * - * @return A state handle that can create an input stream producing the data written to this stream. - * @throws IOException Thrown, if the stream cannot be closed. - */ - public abstract StreamStateHandle closeAndGetHandle() throws IOException; - } - - /** - * A dedicated DataOutputView stream that produces a {@code StateHandle} when closed. - */ - public static final class CheckpointStateOutputView extends DataOutputViewStreamWrapper { - - private final CheckpointStateOutputStream out; - - public CheckpointStateOutputView(CheckpointStateOutputStream out) { - super(out); - this.out = out; - } - - /** - * Closes the stream and gets a state handle that can create a DataInputView. - * producing the data written to this stream. - * - * @return A state handle that can create an input stream producing the data written to this stream. - * @throws IOException Thrown, if the stream cannot be closed. - */ - public StateHandle closeAndGetHandle() throws IOException { - return new DataInputViewHandle(out.closeAndGetHandle()); - } - - @Override - public void close() throws IOException { - out.close(); - } - } - - /** - * Simple state handle that resolved a {@link DataInputView} from a StreamStateHandle. - */ - private static final class DataInputViewHandle implements StateHandle { - - private static final long serialVersionUID = 2891559813513532079L; - - private final StreamStateHandle stream; - - private DataInputViewHandle(StreamStateHandle stream) { - this.stream = stream; - } - - @Override - public DataInputView getState(ClassLoader userCodeClassLoader) throws Exception { - return new DataInputViewStreamWrapper(stream.getState(userCodeClassLoader)); - } - - @Override - public void discardState() throws Exception { - stream.discardState(); - } - - @Override - public long getStateSize() throws Exception { - return stream.getStateSize(); - } - } -} diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/StateBackendFactory.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/StateBackendFactory.java index 5b622ebe5e533..f17eb6e5bc448 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/state/StateBackendFactory.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/StateBackendFactory.java @@ -26,7 +26,7 @@ * * @param The type of the state backend created. */ -public interface StateBackendFactory> { +public interface StateBackendFactory { /** * Creates the state backend, optionally using the given configuration. @@ -36,5 +36,5 @@ public interface StateBackendFactory> { * * @throws Exception Exceptions during instantiation can be forwarded. */ - StateBackend createFromConfig(Configuration config) throws Exception; + AbstractStateBackend createFromConfig(Configuration config) throws Exception; } diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/filesystem/AbstractFileState.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/filesystem/AbstractFileStateHandle.java similarity index 95% rename from flink-runtime/src/main/java/org/apache/flink/runtime/state/filesystem/AbstractFileState.java rename to flink-runtime/src/main/java/org/apache/flink/runtime/state/filesystem/AbstractFileStateHandle.java index 8c2b12af7c344..00800b2727c5d 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/state/filesystem/AbstractFileState.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/filesystem/AbstractFileStateHandle.java @@ -28,7 +28,7 @@ /** * Base class for state that is stored in a file. */ -public abstract class AbstractFileState implements java.io.Serializable { +public abstract class AbstractFileStateHandle implements java.io.Serializable { private static final long serialVersionUID = 350284443258002355L; @@ -43,7 +43,7 @@ public abstract class AbstractFileState implements java.io.Serializable { * * @param filePath The path to the file that stores the state. */ - protected AbstractFileState(Path filePath) { + protected AbstractFileStateHandle(Path filePath) { this.filePath = requireNonNull(filePath); } diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/filesystem/AbstractFsState.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/filesystem/AbstractFsState.java new file mode 100644 index 0000000000000..5035953a4c9ca --- /dev/null +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/filesystem/AbstractFsState.java @@ -0,0 +1,95 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.runtime.state.filesystem; + +import org.apache.flink.api.common.state.ListState; +import org.apache.flink.api.common.state.State; +import org.apache.flink.api.common.state.StateDescriptor; +import org.apache.flink.api.common.typeutils.TypeSerializer; +import org.apache.flink.core.fs.Path; +import org.apache.flink.core.memory.DataOutputViewStreamWrapper; +import org.apache.flink.runtime.state.AbstractHeapState; +import org.apache.flink.runtime.state.KvStateSnapshot; + +import java.io.DataOutputStream; +import java.util.HashMap; +import java.util.Map; + +/** + * Base class for partitioned {@link ListState} implementations that are backed by a regular + * heap hash map. The concrete implementations define how the state is checkpointed. + * + * @param The type of the key. + * @param The type of the namespace. + * @param The type of the values in the state. + * @param The type of State + * @param The type of StateDescriptor for the State S + */ +public abstract class AbstractFsState> + extends AbstractHeapState { + + /** The file system state backend backing snapshots of this state */ + private final FsStateBackend backend; + + public AbstractFsState(FsStateBackend backend, + TypeSerializer keySerializer, + TypeSerializer namespaceSerializer, + TypeSerializer stateSerializer, + SD stateDesc) { + super(keySerializer, namespaceSerializer, stateSerializer, stateDesc); + this.backend = backend; + } + + public AbstractFsState(FsStateBackend backend, + TypeSerializer keySerializer, + TypeSerializer namespaceSerializer, + TypeSerializer stateSerializer, + SD stateDesc, + HashMap> state) { + super(keySerializer, namespaceSerializer, stateSerializer, stateDesc, state); + this.backend = backend; + } + + public abstract KvStateSnapshot createHeapSnapshot(Path filePath); + + @Override + public KvStateSnapshot snapshot(long checkpointId, long timestamp) throws Exception { + + try (FsStateBackend.FsCheckpointStateOutputStream out = backend.createCheckpointStateOutputStream(checkpointId, timestamp)) { + + // serialize the state to the output stream + DataOutputViewStreamWrapper outView = new DataOutputViewStreamWrapper(new DataOutputStream(out)); + outView.writeInt(state.size()); + for (Map.Entry> namespaceState: state.entrySet()) { + N namespace = namespaceState.getKey(); + namespaceSerializer.serialize(namespace, outView); + outView.writeInt(namespaceState.getValue().size()); + for (Map.Entry entry: namespaceState.getValue().entrySet()) { + keySerializer.serialize(entry.getKey(), outView); + stateSerializer.serialize(entry.getValue(), outView); + } + } + outView.flush(); + + // create a handle to the state +// return new FsHeapValueStateSnapshot<>(getKeySerializer(), getNamespaceSerializer(), stateDesc, out.closeAndGetPath()); + return createHeapSnapshot(out.closeAndGetPath()); + } + } +} diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/filesystem/AbstractFsStateSnapshot.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/filesystem/AbstractFsStateSnapshot.java new file mode 100644 index 0000000000000..c1e0f12441d71 --- /dev/null +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/filesystem/AbstractFsStateSnapshot.java @@ -0,0 +1,136 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.runtime.state.filesystem; + +import org.apache.flink.api.common.state.State; +import org.apache.flink.api.common.state.StateDescriptor; +import org.apache.flink.api.common.typeutils.TypeSerializer; +import org.apache.flink.core.fs.FSDataInputStream; +import org.apache.flink.core.fs.Path; +import org.apache.flink.core.memory.DataInputViewStreamWrapper; +import org.apache.flink.runtime.state.KvState; +import org.apache.flink.runtime.state.KvStateSnapshot; + +import java.io.DataInputStream; +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +/** + * A snapshot of a heap key/value state stored in a file. + * + * @param The type of the key in the snapshot state. + * @param The type of the namespace in the snapshot state. + * @param The type of the state value. + */ +public abstract class AbstractFsStateSnapshot> extends AbstractFileStateHandle implements KvStateSnapshot { + + private static final long serialVersionUID = 1L; + + /** Key Serializer */ + protected final TypeSerializer keySerializer; + + /** Namespace Serializer */ + protected final TypeSerializer namespaceSerializer; + + /** Serializer for the state value */ + protected final TypeSerializer stateSerializer; + + /** StateDescriptor, for sanity checks */ + protected final SD stateDesc; + + /** + * Creates a new state snapshot with data in the file system. + * + * @param keySerializer The serializer for the keys. + * @param namespaceSerializer The serializer for the namespace. + * @param stateSerializer The serializer for the elements in the state HashMap + * @param stateDesc The state identifier + * @param filePath The path where the snapshot data is stored. + */ + public AbstractFsStateSnapshot(TypeSerializer keySerializer, + TypeSerializer namespaceSerializer, + TypeSerializer stateSerializer, + SD stateDesc, + Path filePath) { + super(filePath); + this.stateDesc = stateDesc; + this.keySerializer = keySerializer; + this.stateSerializer = stateSerializer; + this.namespaceSerializer = namespaceSerializer; + + } + + public abstract KvState createFsState(FsStateBackend backend, HashMap> stateMap); + + @Override + public KvState restoreState( + FsStateBackend stateBackend, + final TypeSerializer keySerializer, + ClassLoader classLoader, + long recoveryTimestamp) throws Exception { + + // validity checks + if (!this.keySerializer.equals(keySerializer)) { + throw new IllegalArgumentException( + "Cannot restore the state from the snapshot with the given serializers. " + + "State (K/V) was serialized with " + + "(" + this.keySerializer + ") " + + "now is (" + keySerializer + ")"); + } + + // state restore + try (FSDataInputStream inStream = stateBackend.getFileSystem().open(getFilePath())) { + DataInputViewStreamWrapper inView = new DataInputViewStreamWrapper(new DataInputStream(inStream)); + + + final int numKeys = inView.readInt(); + HashMap> stateMap = new HashMap<>(numKeys); + + for (int i = 0; i < numKeys; i++) { + N namespace = namespaceSerializer.deserialize(inView); + final int numValues = inView.readInt(); + Map namespaceMap = new HashMap<>(numValues); + stateMap.put(namespace, namespaceMap); + for (int j = 0; j < numValues; j++) { + K key = keySerializer.deserialize(inView); + SV value = stateSerializer.deserialize(inView); + namespaceMap.put(key, value); + } + } + +// return new FsHeapValueState<>(stateBackend, keySerializer, namespaceSerializer, stateDesc, stateMap); + return createFsState(stateBackend, stateMap); + } + catch (Exception e) { + throw new Exception("Failed to restore state from file system", e); + } + } + + /** + * Returns the file size in bytes. + * + * @return The file size in bytes. + * @throws IOException Thrown if the file system cannot be accessed. + */ + @Override + public long getStateSize() throws IOException { + return getFileSize(); + } +} diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/filesystem/FileSerializableStateHandle.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/filesystem/FileSerializableStateHandle.java index 456f2f2b9584b..662678e58cc12 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/state/filesystem/FileSerializableStateHandle.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/filesystem/FileSerializableStateHandle.java @@ -32,7 +32,7 @@ * * @param The type of state pointed to by the state handle. */ -public class FileSerializableStateHandle extends AbstractFileState implements StateHandle { +public class FileSerializableStateHandle extends AbstractFileStateHandle implements StateHandle { private static final long serialVersionUID = -657631394290213622L; diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/filesystem/FileStreamStateHandle.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/filesystem/FileStreamStateHandle.java index 3b060b5c90130..be9c4cd251785 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/state/filesystem/FileStreamStateHandle.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/filesystem/FileStreamStateHandle.java @@ -29,7 +29,7 @@ /** * A state handle that points to state in a file system, accessible as an input stream. */ -public class FileStreamStateHandle extends AbstractFileState implements StreamStateHandle { +public class FileStreamStateHandle extends AbstractFileStateHandle implements StreamStateHandle { private static final long serialVersionUID = -6826990484549987311L; diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/filesystem/FsHeapKvState.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/filesystem/FsHeapKvState.java deleted file mode 100644 index a1c77820e1628..0000000000000 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/state/filesystem/FsHeapKvState.java +++ /dev/null @@ -1,86 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.runtime.state.filesystem; - -import org.apache.flink.api.common.typeutils.TypeSerializer; -import org.apache.flink.core.memory.DataOutputViewStreamWrapper; -import org.apache.flink.runtime.state.AbstractHeapKvState; - -import java.util.HashMap; - -/** - * Heap-backed key/value state that is snapshotted into files. - * - * @param The type of the key. - * @param The type of the value. - */ -public class FsHeapKvState extends AbstractHeapKvState { - - /** The file system state backend backing snapshots of this state */ - private final FsStateBackend backend; - - /** - * Creates a new and empty key/value state. - * - * @param keySerializer The serializer for the key. - * @param valueSerializer The serializer for the value. - * @param defaultValue The value that is returned when no other value has been associated with a key, yet. - * @param backend The file system state backend backing snapshots of this state - */ - public FsHeapKvState(TypeSerializer keySerializer, TypeSerializer valueSerializer, - V defaultValue, FsStateBackend backend) { - super(keySerializer, valueSerializer, defaultValue); - this.backend = backend; - } - - /** - * Creates a new key/value state with the given state contents. - * This method is used to re-create key/value state with existing data, for example from - * a snapshot. - * - * @param keySerializer The serializer for the key. - * @param valueSerializer The serializer for the value. - * @param defaultValue The value that is returned when no other value has been associated with a key, yet. - * @param state The map of key/value pairs to initialize the state with. - * @param backend The file system state backend backing snapshots of this state - */ - public FsHeapKvState(TypeSerializer keySerializer, TypeSerializer valueSerializer, - V defaultValue, HashMap state, FsStateBackend backend) { - super(keySerializer, valueSerializer, defaultValue, state); - this.backend = backend; - } - - - @Override - public FsHeapKvStateSnapshot snapshot(long checkpointId, long timestamp) throws Exception { - // first, create an output stream to write to - try (FsStateBackend.FsCheckpointStateOutputStream out = - backend.createCheckpointStateOutputStream(checkpointId, timestamp)) { - - // serialize the state to the output stream - DataOutputViewStreamWrapper outView = new DataOutputViewStreamWrapper(out); - outView.writeInt(size()); - writeStateToOutputView(outView); - outView.flush(); - - // create a handle to the state - return new FsHeapKvStateSnapshot<>(getKeySerializer(), getValueSerializer(), out.closeAndGetPath()); - } - } -} diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/filesystem/FsHeapKvStateSnapshot.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/filesystem/FsHeapKvStateSnapshot.java deleted file mode 100644 index 9c8663a96b04d..0000000000000 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/state/filesystem/FsHeapKvStateSnapshot.java +++ /dev/null @@ -1,107 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.runtime.state.filesystem; - -import org.apache.flink.api.common.typeutils.TypeSerializer; -import org.apache.flink.core.fs.FSDataInputStream; -import org.apache.flink.core.fs.Path; -import org.apache.flink.core.memory.DataInputViewStreamWrapper; -import org.apache.flink.runtime.state.KvStateSnapshot; - -import java.io.IOException; -import java.util.HashMap; - -/** - * A snapshot of a heap key/value state stored in a file. - * - * @param The type of the key in the snapshot state. - * @param The type of the value in the snapshot state. - */ -public class FsHeapKvStateSnapshot extends AbstractFileState implements KvStateSnapshot { - - private static final long serialVersionUID = 1L; - - /** Name of the key serializer class */ - private final String keySerializerClassName; - - /** Name of the value serializer class */ - private final String valueSerializerClassName; - - /** - * Creates a new state snapshot with data in the file system. - * - * @param keySerializer The serializer for the keys. - * @param valueSerializer The serializer for the values. - * @param filePath The path where the snapshot data is stored. - */ - public FsHeapKvStateSnapshot(TypeSerializer keySerializer, TypeSerializer valueSerializer, Path filePath) { - super(filePath); - this.keySerializerClassName = keySerializer.getClass().getName(); - this.valueSerializerClassName = valueSerializer.getClass().getName(); - } - - @Override - public FsHeapKvState restoreState( - FsStateBackend stateBackend, - final TypeSerializer keySerializer, - final TypeSerializer valueSerializer, - V defaultValue, - ClassLoader classLoader, - long recoveryTimestamp) throws Exception { - - // validity checks - if (!keySerializer.getClass().getName().equals(keySerializerClassName) || - !valueSerializer.getClass().getName().equals(valueSerializerClassName)) { - throw new IllegalArgumentException( - "Cannot restore the state from the snapshot with the given serializers. " + - "State (K/V) was serialized with (" + valueSerializerClassName + - "/" + keySerializerClassName + ")"); - } - - // state restore - try (FSDataInputStream inStream = stateBackend.getFileSystem().open(getFilePath())) { - DataInputViewStreamWrapper inView = new DataInputViewStreamWrapper(inStream); - - final int numEntries = inView.readInt(); - HashMap stateMap = new HashMap<>(numEntries); - - for (int i = 0; i < numEntries; i++) { - K key = keySerializer.deserialize(inView); - V value = valueSerializer.deserialize(inView); - stateMap.put(key, value); - } - - return new FsHeapKvState(keySerializer, valueSerializer, defaultValue, stateMap, stateBackend); - } - catch (Exception e) { - throw new Exception("Failed to restore state from file system", e); - } - } - - /** - * Returns the file size in bytes. - * - * @return The file size in bytes. - * @throws IOException Thrown if the file system cannot be accessed. - */ - @Override - public long getStateSize() throws IOException { - return getFileSize(); - } -} diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/filesystem/FsListState.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/filesystem/FsListState.java new file mode 100644 index 0000000000000..1d5b5f89381cc --- /dev/null +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/filesystem/FsListState.java @@ -0,0 +1,140 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.runtime.state.filesystem; + +import org.apache.flink.api.common.state.ListStateDescriptor; +import org.apache.flink.api.common.state.ListState; +import org.apache.flink.api.common.typeutils.TypeSerializer; +import org.apache.flink.core.fs.Path; +import org.apache.flink.runtime.state.ArrayListSerializer; +import org.apache.flink.runtime.state.KvState; +import org.apache.flink.runtime.state.KvStateSnapshot; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * Heap-backed partitioned {@link org.apache.flink.api.common.state.ListState} that is snapshotted + * into files. + * + * @param The type of the key. + * @param The type of the namespace. + * @param The type of the value. + */ +public class FsListState + extends AbstractFsState, ListState, ListStateDescriptor> + implements ListState { + + /** + * Creates a new and empty partitioned state. + * + * @param keySerializer The serializer for the key. + * @param stateDesc The state identifier for the state. This contains name + * and can create a default state value. + * @param backend The file system state backend backing snapshots of this state + */ + public FsListState(FsStateBackend backend, + TypeSerializer keySerializer, + TypeSerializer namespaceSerializer, + ListStateDescriptor stateDesc) { + super(backend, keySerializer, namespaceSerializer, new ArrayListSerializer<>(stateDesc.getSerializer()), stateDesc); + } + + /** + * Creates a new key/value state with the given state contents. + * This method is used to re-create key/value state with existing data, for example from + * a snapshot. + * + * @param keySerializer The serializer for the key. + * @param namespaceSerializer The serializer for the namespace. + * @param stateDesc The state identifier for the state. This contains name + * and can create a default state value. + * @param state The map of key/value pairs to initialize the state with. + * @param backend The file system state backend backing snapshots of this state + */ + public FsListState(FsStateBackend backend, + TypeSerializer keySerializer, + TypeSerializer namespaceSerializer, + ListStateDescriptor stateDesc, + HashMap>> state) { + super(backend, keySerializer, namespaceSerializer, new ArrayListSerializer<>(stateDesc.getSerializer()), stateDesc, state); + } + + + @Override + public Iterable get() { + if (currentNSState == null) { + currentNSState = state.get(currentNamespace); + } + if (currentNSState != null) { + List result = currentNSState.get(currentKey); + if (result == null) { + return Collections.emptyList(); + } else { + return result; + } + } + return Collections.emptyList(); + } + + @Override + public void add(V value) { + if (currentKey == null) { + throw new RuntimeException("No key available."); + } + + if (currentNSState == null) { + currentNSState = new HashMap<>(); + state.put(currentNamespace, currentNSState); + } + + + ArrayList list = currentNSState.get(currentKey); + if (list == null) { + list = new ArrayList<>(); + currentNSState.put(currentKey, list); + } + list.add(value); + } + + @Override + public KvStateSnapshot, ListStateDescriptor, FsStateBackend> createHeapSnapshot(Path filePath) { + return new Snapshot<>(getKeySerializer(), getNamespaceSerializer(), new ArrayListSerializer<>(stateDesc.getSerializer()), stateDesc, filePath); + } + + public static class Snapshot extends AbstractFsStateSnapshot, ListState, ListStateDescriptor> { + private static final long serialVersionUID = 1L; + + public Snapshot(TypeSerializer keySerializer, + TypeSerializer namespaceSerializer, + TypeSerializer> stateSerializer, + ListStateDescriptor stateDescs, + Path filePath) { + super(keySerializer, namespaceSerializer, stateSerializer, stateDescs, filePath); + } + + @Override + public KvState, ListStateDescriptor, FsStateBackend> createFsState(FsStateBackend backend, HashMap>> stateMap) { + return new FsListState<>(backend, keySerializer, namespaceSerializer, stateDesc, stateMap); + } + } +} diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/filesystem/FsReducingState.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/filesystem/FsReducingState.java new file mode 100644 index 0000000000000..ef721c973c01c --- /dev/null +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/filesystem/FsReducingState.java @@ -0,0 +1,149 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.runtime.state.filesystem; + +import org.apache.flink.api.common.functions.ReduceFunction; +import org.apache.flink.api.common.state.ReducingState; +import org.apache.flink.api.common.state.ReducingStateDescriptor; +import org.apache.flink.api.common.typeutils.TypeSerializer; +import org.apache.flink.core.fs.Path; +import org.apache.flink.runtime.state.KvState; +import org.apache.flink.runtime.state.KvStateSnapshot; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +/** + * Heap-backed partitioned {@link org.apache.flink.api.common.state.ReducingState} that is + * snapshotted into files. + * + * @param The type of the key. + * @param The type of the namespace. + * @param The type of the value. + */ +public class FsReducingState + extends AbstractFsState, ReducingStateDescriptor> + implements ReducingState { + + private final ReduceFunction reduceFunction; + + /** + * Creates a new and empty partitioned state. + * + * @param backend The file system state backend backing snapshots of this state + * @param keySerializer The serializer for the key. + * @param namespaceSerializer The serializer for the namespace. + * @param stateDesc The state identifier for the state. This contains name + * and can create a default state value. + */ + public FsReducingState(FsStateBackend backend, + TypeSerializer keySerializer, + TypeSerializer namespaceSerializer, + ReducingStateDescriptor stateDesc) { + super(backend, keySerializer, namespaceSerializer, stateDesc.getSerializer(), stateDesc); + this.reduceFunction = stateDesc.getReduceFunction(); + } + + /** + * Creates a new key/value state with the given state contents. + * This method is used to re-create key/value state with existing data, for example from + * a snapshot. + * + * @param backend The file system state backend backing snapshots of this state + * @param keySerializer The serializer for the key. + * @param namespaceSerializer The serializer for the namespace. + * @param stateDesc The state identifier for the state. This contains name +* and can create a default state value. + * @param state The map of key/value pairs to initialize the state with. + */ + public FsReducingState(FsStateBackend backend, + TypeSerializer keySerializer, + TypeSerializer namespaceSerializer, + ReducingStateDescriptor stateDesc, + HashMap> state) { + super(backend, keySerializer, namespaceSerializer, stateDesc.getSerializer(), stateDesc, state); + this.reduceFunction = stateDesc.getReduceFunction(); + } + + + @Override + public V get() { + if (currentNSState == null) { + currentNSState = state.get(currentNamespace); + } + if (currentNSState != null) { + return currentNSState.get(currentKey); + } + return null; + } + + @Override + public void add(V value) throws IOException { + if (currentKey == null) { + throw new RuntimeException("No key available."); + } + + if (currentNSState == null) { + currentNSState = new HashMap<>(); + state.put(currentNamespace, currentNSState); + } +// currentKeyState.merge(currentNamespace, value, new BiFunction() { +// @Override +// public V apply(V v, V v2) { +// try { +// return reduceFunction.reduce(v, v2); +// } catch (Exception e) { +// return null; +// } +// } +// }); + V currentValue = currentNSState.get(currentKey); + if (currentValue == null) { + currentNSState.put(currentKey, value); + } else { + try { + currentNSState.put(currentKey, reduceFunction.reduce(currentValue, value)); + } catch (Exception e) { + throw new RuntimeException("Could not add value to reducing state.", e); + } + } + } + @Override + public KvStateSnapshot, ReducingStateDescriptor, FsStateBackend> createHeapSnapshot(Path filePath) { + return new Snapshot<>(getKeySerializer(), getNamespaceSerializer(), stateSerializer, stateDesc, filePath); + } + + public static class Snapshot extends AbstractFsStateSnapshot, ReducingStateDescriptor> { + private static final long serialVersionUID = 1L; + + public Snapshot(TypeSerializer keySerializer, + TypeSerializer namespaceSerializer, + TypeSerializer stateSerializer, + ReducingStateDescriptor stateDescs, + Path filePath) { + super(keySerializer, namespaceSerializer, stateSerializer, stateDescs, filePath); + } + + @Override + public KvState, ReducingStateDescriptor, FsStateBackend> createFsState(FsStateBackend backend, HashMap> stateMap) { + return new FsReducingState<>(backend, keySerializer, namespaceSerializer, stateDesc, stateMap); + } + } +} diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/filesystem/FsStateBackend.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/filesystem/FsStateBackend.java index ed28e5ef45a1e..411b53682ec2d 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/state/filesystem/FsStateBackend.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/filesystem/FsStateBackend.java @@ -18,16 +18,22 @@ package org.apache.flink.runtime.state.filesystem; +import org.apache.flink.api.common.state.ListState; +import org.apache.flink.api.common.state.ListStateDescriptor; +import org.apache.flink.api.common.state.ReducingState; +import org.apache.flink.api.common.state.ReducingStateDescriptor; +import org.apache.flink.api.common.state.ValueState; +import org.apache.flink.api.common.state.ValueStateDescriptor; import org.apache.flink.api.common.typeutils.TypeSerializer; import org.apache.flink.core.fs.FSDataOutputStream; import org.apache.flink.core.fs.FileSystem; import org.apache.flink.core.fs.Path; import org.apache.flink.runtime.execution.Environment; -import org.apache.flink.runtime.state.StateBackend; import org.apache.flink.runtime.state.StateHandle; +import org.apache.flink.runtime.state.AbstractStateBackend; + import org.apache.flink.runtime.state.StreamStateHandle; import org.apache.flink.runtime.state.memory.ByteStreamStateHandle; - import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -48,7 +54,7 @@ * * {@code hdfs://namenode:port/flink-checkpoints//chk-17/6ba7b810-9dad-11d1-80b4-00c04fd430c8 } */ -public class FsStateBackend extends StateBackend { +public class FsStateBackend extends AbstractStateBackend { private static final long serialVersionUID = -8191916350224044011L; @@ -264,7 +270,11 @@ public FileSystem getFileSystem() { // ------------------------------------------------------------------------ @Override - public void initializeForJob(Environment env) throws Exception { + public void initializeForJob(Environment env, + String operatorIdentifier, + TypeSerializer keySerializer) throws Exception { + super.initializeForJob(env, operatorIdentifier, keySerializer); + Path dir = new Path(basePath, env.getJobID().toString()); LOG.info("Initializing file state backend to URI " + dir); @@ -298,11 +308,21 @@ public void close() throws Exception {} // ------------------------------------------------------------------------ @Override - public FsHeapKvState createKvState(String stateId, String stateName, - TypeSerializer keySerializer, TypeSerializer valueSerializer, V defaultValue) throws Exception { - return new FsHeapKvState(keySerializer, valueSerializer, defaultValue, this); + public ValueState createValueState(TypeSerializer namespaceSerializer, ValueStateDescriptor stateDesc) throws Exception { + return new FsValueState<>(this, keySerializer, namespaceSerializer, stateDesc); + } + + @Override + public ListState createListState(TypeSerializer namespaceSerializer, ListStateDescriptor stateDesc) throws Exception { + return new FsListState<>(this, keySerializer, namespaceSerializer, stateDesc); } + @Override + public ReducingState createReducingState(TypeSerializer namespaceSerializer, ReducingStateDescriptor stateDesc) throws Exception { + return new FsReducingState<>(this, keySerializer, namespaceSerializer, stateDesc); + } + + @Override public StateHandle checkpointStateSerializable( S state, long checkpointID, long timestamp) throws Exception diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/filesystem/FsValueState.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/filesystem/FsValueState.java new file mode 100644 index 0000000000000..1a53980991273 --- /dev/null +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/filesystem/FsValueState.java @@ -0,0 +1,126 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.runtime.state.filesystem; + +import org.apache.flink.api.common.state.ValueState; +import org.apache.flink.api.common.state.ValueStateDescriptor; +import org.apache.flink.api.common.typeutils.TypeSerializer; +import org.apache.flink.core.fs.Path; +import org.apache.flink.runtime.state.KvState; +import org.apache.flink.runtime.state.KvStateSnapshot; + +import java.util.HashMap; +import java.util.Map; + +/** + * Heap-backed partitioned {@link org.apache.flink.api.common.state.ValueState} that is snapshotted + * into files. + * + * @param The type of the key. + * @param The type of the namespace. + * @param The type of the value. + */ +public class FsValueState + extends AbstractFsState, ValueStateDescriptor> + implements ValueState { + + /** + * Creates a new and empty key/value state. + * + * @param keySerializer The serializer for the key. + * @param namespaceSerializer The serializer for the namespace. + * @param stateDesc The state identifier for the state. This contains name + * and can create a default state value. + * @param backend The file system state backend backing snapshots of this state + */ + public FsValueState(FsStateBackend backend, + TypeSerializer keySerializer, + TypeSerializer namespaceSerializer, + ValueStateDescriptor stateDesc) { + super(backend, keySerializer, namespaceSerializer, stateDesc.getSerializer(), stateDesc); + } + + /** + * Creates a new key/value state with the given state contents. + * This method is used to re-create key/value state with existing data, for example from + * a snapshot. + * + * @param keySerializer The serializer for the key. + * @param namespaceSerializer The serializer for the namespace. + * @param stateDesc The state identifier for the state. This contains name + * and can create a default state value. + * @param state The map of key/value pairs to initialize the state with. + * @param backend The file system state backend backing snapshots of this state + */ + public FsValueState(FsStateBackend backend, + TypeSerializer keySerializer, + TypeSerializer namespaceSerializer, + ValueStateDescriptor stateDesc, + HashMap> state) { + super(backend, keySerializer, namespaceSerializer, stateDesc.getSerializer(), stateDesc, state); + } + + @Override + public V value() { + if (currentNSState == null) { + currentNSState = state.get(currentNamespace); + } + if (currentNSState != null) { + V value = currentNSState.get(currentKey); + return value != null ? value : stateDesc.getDefaultValue(); + } + return stateDesc.getDefaultValue(); + } + + @Override + public void update(V value) { + if (currentKey == null) { + throw new RuntimeException("No key available."); + } + + if (currentNSState == null) { + currentNSState = new HashMap<>(); + state.put(currentNamespace, currentNSState); + } + + currentNSState.put(currentKey, value); + } + + @Override + public KvStateSnapshot, ValueStateDescriptor, FsStateBackend> createHeapSnapshot(Path filePath) { + return new Snapshot<>(getKeySerializer(), getNamespaceSerializer(), stateSerializer, stateDesc, filePath); + } + + public static class Snapshot extends AbstractFsStateSnapshot, ValueStateDescriptor> { + private static final long serialVersionUID = 1L; + + public Snapshot(TypeSerializer keySerializer, + TypeSerializer namespaceSerializer, + TypeSerializer stateSerializer, + ValueStateDescriptor stateDescs, + Path filePath) { + super(keySerializer, namespaceSerializer, stateSerializer, stateDescs, filePath); + } + + @Override + public KvState, ValueStateDescriptor, FsStateBackend> createFsState(FsStateBackend backend, HashMap> stateMap) { + return new FsValueState<>(backend, keySerializer, namespaceSerializer, stateDesc, stateMap); + } + } +} diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/memory/AbstractMemState.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/memory/AbstractMemState.java new file mode 100644 index 0000000000000..816c883f5e185 --- /dev/null +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/memory/AbstractMemState.java @@ -0,0 +1,82 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.runtime.state.memory; + +import org.apache.flink.api.common.state.ListState; +import org.apache.flink.api.common.state.State; +import org.apache.flink.api.common.state.StateDescriptor; +import org.apache.flink.api.common.typeutils.TypeSerializer; +import org.apache.flink.runtime.state.AbstractHeapState; +import org.apache.flink.runtime.state.KvStateSnapshot; +import org.apache.flink.runtime.util.DataOutputSerializer; + +import java.util.HashMap; +import java.util.Map; + +/** + * Base class for partitioned {@link ListState} implementations that are backed by a regular + * heap hash map. The concrete implementations define how the state is checkpointed. + * + * @param The type of the key. + * @param The type of the namespace. + * @param The type of the values in the state. + * @param The type of State + * @param The type of StateDescriptor for the State S + */ +public abstract class AbstractMemState> + extends AbstractHeapState { + + public AbstractMemState(TypeSerializer keySerializer, + TypeSerializer namespaceSerializer, + TypeSerializer stateSerializer, + SD stateDesc) { + super(keySerializer, namespaceSerializer, stateSerializer, stateDesc); + } + + public AbstractMemState(TypeSerializer keySerializer, + TypeSerializer namespaceSerializer, + TypeSerializer stateSerializer, + SD stateDesc, + HashMap> state) { + super(keySerializer, namespaceSerializer, stateSerializer, stateDesc, state); + } + + public abstract KvStateSnapshot createHeapSnapshot(byte[] bytes); + + @Override + public KvStateSnapshot snapshot(long checkpointId, long timestamp) throws Exception { + + DataOutputSerializer out = new DataOutputSerializer(Math.max(size() * 16, 16)); + + out.writeInt(state.size()); + for (Map.Entry> namespaceState: state.entrySet()) { + N namespace = namespaceState.getKey(); + namespaceSerializer.serialize(namespace, out); + out.writeInt(namespaceState.getValue().size()); + for (Map.Entry entry: namespaceState.getValue().entrySet()) { + keySerializer.serialize(entry.getKey(), out); + stateSerializer.serialize(entry.getValue(), out); + } + } + + byte[] bytes = out.getCopyOfBuffer(); + + return createHeapSnapshot(bytes); + } +} diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/memory/AbstractMemStateSnapshot.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/memory/AbstractMemStateSnapshot.java new file mode 100644 index 0000000000000..d2efd53c54f30 --- /dev/null +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/memory/AbstractMemStateSnapshot.java @@ -0,0 +1,127 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.runtime.state.memory; + +import org.apache.flink.api.common.state.State; +import org.apache.flink.api.common.state.StateDescriptor; +import org.apache.flink.api.common.typeutils.TypeSerializer; +import org.apache.flink.runtime.state.KvState; +import org.apache.flink.runtime.state.KvStateSnapshot; +import org.apache.flink.runtime.util.DataInputDeserializer; + +import java.util.HashMap; +import java.util.Map; + +/** + * A snapshot of a {@link MemValueState} for a checkpoint. The data is stored in a heap byte + * array, in serialized form. + * + * @param The type of the key in the snapshot state. + * @param The type of the namespace in the snapshot state. + * @param The type of the value in the snapshot state. + */ +public abstract class AbstractMemStateSnapshot> implements KvStateSnapshot { + + private static final long serialVersionUID = 1L; + + /** Key Serializer */ + protected final TypeSerializer keySerializer; + + /** Namespace Serializer */ + protected final TypeSerializer namespaceSerializer; + + /** Serializer for the state value */ + protected final TypeSerializer stateSerializer; + + /** StateDescriptor, for sanity checks */ + protected final SD stateDesc; + + /** The serialized data of the state key/value pairs */ + private final byte[] data; + + /** + * Creates a new heap memory state snapshot. + * + * @param keySerializer The serializer for the keys. + * @param namespaceSerializer The serializer for the namespace. + * @param stateSerializer The serializer for the elements in the state HashMap + * @param stateDesc The state identifier + * @param data The serialized data of the state key/value pairs + */ + public AbstractMemStateSnapshot(TypeSerializer keySerializer, + TypeSerializer namespaceSerializer, + TypeSerializer stateSerializer, + SD stateDesc, + byte[] data) { + this.keySerializer = keySerializer; + this.namespaceSerializer = namespaceSerializer; + this.stateSerializer = stateSerializer; + this.stateDesc = stateDesc; + this.data = data; + } + + public abstract KvState createMemState(HashMap> stateMap); + + @Override + public KvState restoreState( + MemoryStateBackend stateBackend, + final TypeSerializer keySerializer, + ClassLoader classLoader, long recoveryTimestamp) throws Exception { + + // validity checks + if (!this.keySerializer.equals(keySerializer)) { + throw new IllegalArgumentException( + "Cannot restore the state from the snapshot with the given serializers. " + + "State (K/V) was serialized with " + + "(" + this.keySerializer + ") " + + "now is (" + keySerializer + ")"); + } + + // restore state + DataInputDeserializer inView = new DataInputDeserializer(data, 0, data.length); + + final int numKeys = inView.readInt(); + HashMap> stateMap = new HashMap<>(numKeys); + + for (int i = 0; i < numKeys; i++) { + N namespace = namespaceSerializer.deserialize(inView); + final int numValues = inView.readInt(); + Map namespaceMap = new HashMap<>(numValues); + stateMap.put(namespace, namespaceMap); + for (int j = 0; j < numValues; j++) { + K key = keySerializer.deserialize(inView); + SV value = stateSerializer.deserialize(inView); + namespaceMap.put(key, value); + } + } + + return createMemState(stateMap); + } + + /** + * Discarding the heap state is a no-op. + */ + @Override + public void discardState() {} + + @Override + public long getStateSize() { + return data.length; + } +} diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/memory/MemHeapKvState.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/memory/MemHeapKvState.java deleted file mode 100644 index 082cb9a94b1db..0000000000000 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/state/memory/MemHeapKvState.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.runtime.state.memory; - -import org.apache.flink.api.common.typeutils.TypeSerializer; -import org.apache.flink.runtime.util.DataOutputSerializer; -import org.apache.flink.runtime.state.AbstractHeapKvState; - -import java.util.HashMap; - -/** - * Heap-backed key/value state that is snapshotted into a serialized memory copy. - * - * @param The type of the key. - * @param The type of the value. - */ -public class MemHeapKvState extends AbstractHeapKvState { - - public MemHeapKvState(TypeSerializer keySerializer, TypeSerializer valueSerializer, V defaultValue) { - super(keySerializer, valueSerializer, defaultValue); - } - - public MemHeapKvState(TypeSerializer keySerializer, TypeSerializer valueSerializer, - V defaultValue, HashMap state) { - super(keySerializer, valueSerializer, defaultValue, state); - } - - @Override - public MemoryHeapKvStateSnapshot snapshot(long checkpointId, long timestamp) throws Exception { - DataOutputSerializer ser = new DataOutputSerializer(Math.max(size() * 16, 16)); - writeStateToOutputView(ser); - byte[] bytes = ser.getCopyOfBuffer(); - - return new MemoryHeapKvStateSnapshot(getKeySerializer(), getValueSerializer(), bytes, size()); - } -} diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/memory/MemListState.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/memory/MemListState.java new file mode 100644 index 0000000000000..d5e4dfd96266c --- /dev/null +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/memory/MemListState.java @@ -0,0 +1,111 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.runtime.state.memory; + +import org.apache.flink.api.common.state.ListState; +import org.apache.flink.api.common.state.ListStateDescriptor; +import org.apache.flink.api.common.typeutils.TypeSerializer; +import org.apache.flink.runtime.state.ArrayListSerializer; +import org.apache.flink.runtime.state.KvState; +import org.apache.flink.runtime.state.KvStateSnapshot; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * Heap-backed partitioned {@link org.apache.flink.api.common.state.ListState} that is snapshotted + * into a serialized memory copy. + * + * @param The type of the key. + * @param The type of the namespace. + * @param The type of the values in the list state. + */ +public class MemListState + extends AbstractMemState, ListState, ListStateDescriptor> + implements ListState { + + public MemListState(TypeSerializer keySerializer, TypeSerializer namespaceSerializer, ListStateDescriptor stateDesc) { + super(keySerializer, namespaceSerializer, new ArrayListSerializer<>(stateDesc.getSerializer()), stateDesc); + } + + public MemListState(TypeSerializer keySerializer, TypeSerializer namespaceSerializer, ListStateDescriptor stateDesc, HashMap>> state) { + super(keySerializer, namespaceSerializer, new ArrayListSerializer<>(stateDesc.getSerializer()), stateDesc, state); + } + + @Override + public Iterable get() { + if (currentNSState == null) { + currentNSState = state.get(currentNamespace); + } + if (currentNSState != null) { + List result = currentNSState.get(currentKey); + if (result == null) { + return Collections.emptyList(); + } else { + return result; + } + } + return Collections.emptyList(); + } + + @Override + public void add(V value) { + if (currentKey == null) { + throw new RuntimeException("No key available."); + } + + if (currentNSState == null) { + currentNSState = new HashMap<>(); + state.put(currentNamespace, currentNSState); + } + + + ArrayList list = currentNSState.get(currentKey); + if (list == null) { + list = new ArrayList<>(); + currentNSState.put(currentKey, list); + } + list.add(value); + } + + @Override + public KvStateSnapshot, ListStateDescriptor, MemoryStateBackend> createHeapSnapshot(byte[] bytes) { + return new Snapshot<>(getKeySerializer(), getNamespaceSerializer(), stateSerializer, stateDesc, bytes); + } + + public static class Snapshot extends AbstractMemStateSnapshot, ListState, ListStateDescriptor> { + private static final long serialVersionUID = 1L; + + public Snapshot(TypeSerializer keySerializer, + TypeSerializer namespaceSerializer, + TypeSerializer> stateSerializer, + ListStateDescriptor stateDescs, byte[] data) { + super(keySerializer, namespaceSerializer, stateSerializer, stateDescs, data); + } + + @Override + public KvState, ListStateDescriptor, MemoryStateBackend> createMemState(HashMap>> stateMap) { + return new MemListState<>(keySerializer, namespaceSerializer, stateDesc, stateMap); + } + } + +} diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/memory/MemReducingState.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/memory/MemReducingState.java new file mode 100644 index 0000000000000..ce1634436b277 --- /dev/null +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/memory/MemReducingState.java @@ -0,0 +1,123 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.runtime.state.memory; + +import org.apache.flink.api.common.functions.ReduceFunction; +import org.apache.flink.api.common.state.ReducingState; +import org.apache.flink.api.common.state.ReducingStateDescriptor; +import org.apache.flink.api.common.typeutils.TypeSerializer; +import org.apache.flink.runtime.state.KvState; +import org.apache.flink.runtime.state.KvStateSnapshot; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +/** + * Heap-backed partitioned {@link org.apache.flink.api.common.state.ReducingState} that is + * snapshotted into a serialized memory copy. + * + * @param The type of the key. + * @param The type of the namespace. + * @param The type of the values in the list state. + */ +public class MemReducingState + extends AbstractMemState, ReducingStateDescriptor> + implements ReducingState { + + private final ReduceFunction reduceFunction; + + public MemReducingState(TypeSerializer keySerializer, + TypeSerializer namespaceSerializer, + ReducingStateDescriptor stateDesc) { + super(keySerializer, namespaceSerializer, stateDesc.getSerializer(), stateDesc); + this.reduceFunction = stateDesc.getReduceFunction(); + } + + public MemReducingState(TypeSerializer keySerializer, + TypeSerializer namespaceSerializer, + ReducingStateDescriptor stateDesc, + HashMap> state) { + super(keySerializer, namespaceSerializer, stateDesc.getSerializer(), stateDesc, state); + this.reduceFunction = stateDesc.getReduceFunction(); + } + + @Override + public V get() { + if (currentNSState == null) { + currentNSState = state.get(currentNamespace); + } + if (currentNSState != null) { + return currentNSState.get(currentKey); + } + return null; + } + + @Override + public void add(V value) throws IOException { + if (currentKey == null) { + throw new RuntimeException("No key available."); + } + + if (currentNSState == null) { + currentNSState = new HashMap<>(); + state.put(currentNamespace, currentNSState); + } +// currentKeyState.merge(currentNamespace, value, new BiFunction() { +// @Override +// public V apply(V v, V v2) { +// try { +// return reduceFunction.reduce(v, v2); +// } catch (Exception e) { +// return null; +// } +// } +// }); + V currentValue = currentNSState.get(currentKey); + if (currentValue == null) { + currentNSState.put(currentKey, value); + } else { + try { + currentNSState.put(currentKey, reduceFunction.reduce(currentValue, value)); + } catch (Exception e) { + throw new RuntimeException("Could not add value to reducing state.", e); + } + } + } + + @Override + public KvStateSnapshot, ReducingStateDescriptor, MemoryStateBackend> createHeapSnapshot(byte[] bytes) { + return new Snapshot<>(getKeySerializer(), getNamespaceSerializer(), stateSerializer, stateDesc, bytes); + } + + public static class Snapshot extends AbstractMemStateSnapshot, ReducingStateDescriptor> { + private static final long serialVersionUID = 1L; + + public Snapshot(TypeSerializer keySerializer, + TypeSerializer namespaceSerializer, + TypeSerializer stateSerializer, + ReducingStateDescriptor stateDescs, byte[] data) { + super(keySerializer, namespaceSerializer, stateSerializer, stateDescs, data); + } + + @Override + public KvState, ReducingStateDescriptor, MemoryStateBackend> createMemState(HashMap> stateMap) { + return new MemReducingState<>(keySerializer, namespaceSerializer, stateDesc, stateMap); + } + }} diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/memory/MemValueState.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/memory/MemValueState.java new file mode 100644 index 0000000000000..8ce166a1a5bd5 --- /dev/null +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/memory/MemValueState.java @@ -0,0 +1,100 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.runtime.state.memory; + +import org.apache.flink.api.common.state.ValueState; +import org.apache.flink.api.common.state.ValueStateDescriptor; +import org.apache.flink.api.common.typeutils.TypeSerializer; +import org.apache.flink.runtime.state.KvState; +import org.apache.flink.runtime.state.KvStateSnapshot; + +import java.util.HashMap; +import java.util.Map; + +/** + * Heap-backed key/value state that is snapshotted into a serialized memory copy. + * + * @param The type of the key. + * @param The type of the namespace. + * @param The type of the value. + */ +public class MemValueState + extends AbstractMemState, ValueStateDescriptor> + implements ValueState { + + public MemValueState(TypeSerializer keySerializer, + TypeSerializer namespaceSerializer, + ValueStateDescriptor stateDesc) { + super(keySerializer, namespaceSerializer, stateDesc.getSerializer(), stateDesc); + } + + public MemValueState(TypeSerializer keySerializer, + TypeSerializer namespaceSerializer, + ValueStateDescriptor stateDesc, + HashMap> state) { + super(keySerializer, namespaceSerializer, stateDesc.getSerializer(), stateDesc, state); + } + + @Override + public V value() { + if (currentNSState == null) { + currentNSState = state.get(currentNamespace); + } + if (currentNSState != null) { + V value = currentNSState.get(currentKey); + return value != null ? value : stateDesc.getDefaultValue(); + } + return stateDesc.getDefaultValue(); + } + + @Override + public void update(V value) { + if (currentKey == null) { + throw new RuntimeException("No key available."); + } + + if (currentNSState == null) { + currentNSState = new HashMap<>(); + state.put(currentNamespace, currentNSState); + } + + currentNSState.put(currentKey, value); + } + + @Override + public KvStateSnapshot, ValueStateDescriptor, MemoryStateBackend> createHeapSnapshot(byte[] bytes) { + return new Snapshot<>(getKeySerializer(), getNamespaceSerializer(), stateSerializer, stateDesc, bytes); + } + + public static class Snapshot extends AbstractMemStateSnapshot, ValueStateDescriptor> { + private static final long serialVersionUID = 1L; + + public Snapshot(TypeSerializer keySerializer, + TypeSerializer namespaceSerializer, + TypeSerializer stateSerializer, + ValueStateDescriptor stateDescs, byte[] data) { + super(keySerializer, namespaceSerializer, stateSerializer, stateDescs, data); + } + + @Override + public KvState, ValueStateDescriptor, MemoryStateBackend> createMemState(HashMap> stateMap) { + return new MemValueState<>(keySerializer, namespaceSerializer, stateDesc, stateMap); + } + } +} diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/memory/MemoryHeapKvStateSnapshot.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/memory/MemoryHeapKvStateSnapshot.java deleted file mode 100644 index 0cb7fa48255ae..0000000000000 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/state/memory/MemoryHeapKvStateSnapshot.java +++ /dev/null @@ -1,107 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.runtime.state.memory; - -import org.apache.flink.api.common.typeutils.TypeSerializer; -import org.apache.flink.runtime.state.KvStateSnapshot; -import org.apache.flink.runtime.util.DataInputDeserializer; - -import java.util.HashMap; - -/** - * A snapshot of a {@link MemHeapKvState} for a checkpoint. The data is stored in a heap byte - * array, in serialized form. - * - * @param The type of the key in the snapshot state. - * @param The type of the value in the snapshot state. - */ -public class MemoryHeapKvStateSnapshot implements KvStateSnapshot { - - private static final long serialVersionUID = 1L; - - /** Name of the key serializer class */ - private final String keySerializerClassName; - - /** Name of the value serializer class */ - private final String valueSerializerClassName; - - /** The serialized data of the state key/value pairs */ - private final byte[] data; - - /** The number of key/value pairs */ - private final int numEntries; - - /** - * Creates a new heap memory state snapshot. - * - * @param keySerializer The serializer for the keys. - * @param valueSerializer The serializer for the values. - * @param data The serialized data of the state key/value pairs - * @param numEntries The number of key/value pairs - */ - public MemoryHeapKvStateSnapshot(TypeSerializer keySerializer, - TypeSerializer valueSerializer, byte[] data, int numEntries) { - this.keySerializerClassName = keySerializer.getClass().getName(); - this.valueSerializerClassName = valueSerializer.getClass().getName(); - this.data = data; - this.numEntries = numEntries; - } - - @Override - public MemHeapKvState restoreState( - MemoryStateBackend stateBackend, - final TypeSerializer keySerializer, - final TypeSerializer valueSerializer, - V defaultValue, - ClassLoader classLoader, - long recoveryTimestamp) throws Exception { - - // validity checks - if (!keySerializer.getClass().getName().equals(keySerializerClassName) || - !valueSerializer.getClass().getName().equals(valueSerializerClassName)) { - throw new IllegalArgumentException( - "Cannot restore the state from the snapshot with the given serializers. " + - "State (K/V) was serialized with (" + valueSerializerClassName + - "/" + keySerializerClassName + ")"); - } - - // restore state - HashMap stateMap = new HashMap<>(numEntries); - DataInputDeserializer in = new DataInputDeserializer(data, 0, data.length); - - for (int i = 0; i < numEntries; i++) { - K key = keySerializer.deserialize(in); - V value = valueSerializer.deserialize(in); - stateMap.put(key, value); - } - - return new MemHeapKvState(keySerializer, valueSerializer, defaultValue, stateMap); - } - - /** - * Discarding the heap state is a no-op. - */ - @Override - public void discardState() {} - - @Override - public long getStateSize() { - return data.length; - } -} diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/memory/MemoryStateBackend.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/memory/MemoryStateBackend.java index 2963237738311..2b7b5f1205841 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/state/memory/MemoryStateBackend.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/memory/MemoryStateBackend.java @@ -18,10 +18,15 @@ package org.apache.flink.runtime.state.memory; +import org.apache.flink.api.common.state.ListState; +import org.apache.flink.api.common.state.ListStateDescriptor; +import org.apache.flink.api.common.state.ReducingState; +import org.apache.flink.api.common.state.ReducingStateDescriptor; +import org.apache.flink.api.common.state.ValueState; +import org.apache.flink.api.common.state.ValueStateDescriptor; import org.apache.flink.api.common.typeutils.TypeSerializer; -import org.apache.flink.runtime.execution.Environment; -import org.apache.flink.runtime.state.StateBackend; import org.apache.flink.runtime.state.StateHandle; +import org.apache.flink.runtime.state.AbstractStateBackend; import org.apache.flink.runtime.state.StreamStateHandle; import java.io.ByteArrayOutputStream; @@ -29,11 +34,11 @@ import java.io.Serializable; /** - * A {@link StateBackend} that stores all its data and checkpoints in memory and has no + * A {@link AbstractStateBackend} that stores all its data and checkpoints in memory and has no * capabilities to spill to disk. Checkpoints are serialized and the serialized data is * transferred */ -public class MemoryStateBackend extends StateBackend { +public class MemoryStateBackend extends AbstractStateBackend { private static final long serialVersionUID = 4109305377809414635L; @@ -65,11 +70,6 @@ public MemoryStateBackend(int maxStateSize) { // initialization and cleanup // ------------------------------------------------------------------------ - @Override - public void initializeForJob(Environment env) { - // nothing to do here - } - @Override public void disposeAllStateForCurrentJob() { // nothing to do here, GC will do it @@ -83,9 +83,18 @@ public void close() throws Exception {} // ------------------------------------------------------------------------ @Override - public MemHeapKvState createKvState(String stateId, String stateName, - TypeSerializer keySerializer, TypeSerializer valueSerializer, V defaultValue) { - return new MemHeapKvState(keySerializer, valueSerializer, defaultValue); + public ValueState createValueState(TypeSerializer namespaceSerializer, ValueStateDescriptor stateDesc) throws Exception { + return new MemValueState<>(keySerializer, namespaceSerializer, stateDesc); + } + + @Override + public ListState createListState(TypeSerializer namespaceSerializer, ListStateDescriptor stateDesc) throws Exception { + return new MemListState<>(keySerializer, namespaceSerializer, stateDesc); + } + + @Override + public ReducingState createReducingState(TypeSerializer namespaceSerializer, ReducingStateDescriptor stateDesc) throws Exception { + return new MemReducingState<>(keySerializer, namespaceSerializer, stateDesc); } /** @@ -196,14 +205,11 @@ public byte[] closeAndGetBytes() throws IOException { // Static default instance // ------------------------------------------------------------------------ - /** The default instance of this state backend, using the default maximal state size */ - private static final MemoryStateBackend DEFAULT_INSTANCE = new MemoryStateBackend(); - /** * Gets the default instance of this state backend, using the default maximal state size. * @return The default instance of this state backend. */ - public static MemoryStateBackend defaultInstance() { - return DEFAULT_INSTANCE; + public static MemoryStateBackend create() { + return new MemoryStateBackend(); } } diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/state/FileStateBackendTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/state/FileStateBackendTest.java index 05bc8fa4da81b..e7bf80eb7edb8 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/state/FileStateBackendTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/state/FileStateBackendTest.java @@ -18,29 +18,8 @@ package org.apache.flink.runtime.state; -import static org.junit.Assert.assertArrayEquals; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; - -import java.io.File; -import java.io.IOException; -import java.io.InputStream; -import java.net.URI; -import java.util.Random; -import java.util.UUID; - import org.apache.commons.io.FileUtils; - -import org.apache.flink.api.common.typeutils.TypeSerializer; -import org.apache.flink.api.common.typeutils.base.FloatSerializer; import org.apache.flink.api.common.typeutils.base.IntSerializer; -import org.apache.flink.api.common.typeutils.base.IntValueSerializer; -import org.apache.flink.api.common.typeutils.base.StringSerializer; -import org.apache.flink.api.java.typeutils.runtime.ValueSerializer; import org.apache.flink.configuration.ConfigConstants; import org.apache.flink.core.fs.Path; import org.apache.flink.core.testutils.CommonTestUtils; @@ -48,12 +27,32 @@ import org.apache.flink.runtime.state.filesystem.FileStreamStateHandle; import org.apache.flink.runtime.state.filesystem.FsStateBackend; import org.apache.flink.runtime.state.memory.ByteStreamStateHandle; -import org.apache.flink.types.IntValue; -import org.apache.flink.types.StringValue; import org.junit.Test; -public class FileStateBackendTest { +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.net.URI; +import java.util.Random; +import java.util.UUID; + +import static org.junit.Assert.*; + +public class FileStateBackendTest extends StateBackendTestBase { + + private File stateDir; + + @Override + protected FsStateBackend getStateBackend() throws Exception { + stateDir = new File(ConfigConstants.DEFAULT_TASK_MANAGER_TMP_PATH, UUID.randomUUID().toString()); + return new FsStateBackend(localFileUri(stateDir)); + } + + @Override + protected void cleanup() throws Exception { + deleteDirectorySilently(stateDir); + } @Test public void testSetupAndSerialization() { @@ -80,7 +79,7 @@ public void testSetupAndSerialization() { // supreme! } - backend.initializeForJob(new DummyEnvironment("test", 1, 0)); + backend.initializeForJob(new DummyEnvironment("test", 1, 0), "test-op", IntSerializer.INSTANCE); assertNotNull(backend.getCheckpointDirectory()); File checkpointDir = new File(backend.getCheckpointDirectory().toUri().getPath()); @@ -105,9 +104,8 @@ public void testSetupAndSerialization() { public void testSerializableState() { File tempDir = new File(ConfigConstants.DEFAULT_TASK_MANAGER_TMP_PATH, UUID.randomUUID().toString()); try { - FsStateBackend backend = CommonTestUtils.createCopySerializable( - new FsStateBackend(tempDir.toURI(), 40)); - backend.initializeForJob(new DummyEnvironment("test", 1, 0)); + FsStateBackend backend = CommonTestUtils.createCopySerializable(new FsStateBackend(localFileUri(tempDir))); + backend.initializeForJob(new DummyEnvironment("test", 1, 0), "test-op", IntSerializer.INSTANCE); File checkpointDir = new File(backend.getCheckpointDirectory().toUri().getPath()); @@ -118,13 +116,13 @@ public void testSerializableState() { StateHandle handle1 = backend.checkpointStateSerializable(state1, 439568923746L, System.currentTimeMillis()); StateHandle handle2 = backend.checkpointStateSerializable(state2, 439568923746L, System.currentTimeMillis()); StateHandle handle3 = backend.checkpointStateSerializable(state3, 439568923746L, System.currentTimeMillis()); - + assertEquals(state1, handle1.getState(getClass().getClassLoader())); handle1.discardState(); - + assertEquals(state2, handle2.getState(getClass().getClassLoader())); handle2.discardState(); - + assertEquals(state3, handle3.getState(getClass().getClassLoader())); handle3.discardState(); @@ -144,10 +142,9 @@ public void testStateOutputStream() { File tempDir = new File(ConfigConstants.DEFAULT_TASK_MANAGER_TMP_PATH, UUID.randomUUID().toString()); try { // the state backend has a very low in-mem state threshold (15 bytes) - FsStateBackend backend = CommonTestUtils.createCopySerializable( - new FsStateBackend(tempDir.toURI(), 15)); - - backend.initializeForJob(new DummyEnvironment("test", 1, 0)); + FsStateBackend backend = CommonTestUtils.createCopySerializable(new FsStateBackend(tempDir.toURI(), 15)); + + backend.initializeForJob(new DummyEnvironment("test", 1, 0), "test-op", IntSerializer.INSTANCE); File checkpointDir = new File(backend.getCheckpointDirectory().toUri().getPath()); @@ -181,14 +178,14 @@ public void testStateOutputStream() { // use with try-with-resources FileStreamStateHandle handle4; - try (StateBackend.CheckpointStateOutputStream stream4 = + try (AbstractStateBackend.CheckpointStateOutputStream stream4 = backend.createCheckpointStateOutputStream(checkpointId, System.currentTimeMillis())) { stream4.write(state4); handle4 = (FileStreamStateHandle) stream4.closeAndGetHandle(); } // close before accessing handle - StateBackend.CheckpointStateOutputStream stream5 = + AbstractStateBackend.CheckpointStateOutputStream stream5 = backend.createCheckpointStateOutputStream(checkpointId, System.currentTimeMillis()); stream5.write(state4); stream5.close(); @@ -223,197 +220,6 @@ public void testStateOutputStream() { } } - @Test - public void testKeyValueState() { - File tempDir = new File(ConfigConstants.DEFAULT_TASK_MANAGER_TMP_PATH, UUID.randomUUID().toString()); - try { - FsStateBackend backend = CommonTestUtils.createCopySerializable(new FsStateBackend(localFileUri(tempDir))); - backend.initializeForJob(new DummyEnvironment("test", 1, 0)); - - File checkpointDir = new File(backend.getCheckpointDirectory().toUri().getPath()); - - KvState kv = - backend.createKvState("0", "a", IntSerializer.INSTANCE, StringSerializer.INSTANCE, null); - - assertEquals(0, kv.size()); - - // some modifications to the state - kv.setCurrentKey(1); - assertNull(kv.value()); - kv.update("1"); - assertEquals(1, kv.size()); - kv.setCurrentKey(2); - assertNull(kv.value()); - kv.update("2"); - assertEquals(2, kv.size()); - kv.setCurrentKey(1); - assertEquals("1", kv.value()); - assertEquals(2, kv.size()); - - // draw a snapshot - KvStateSnapshot snapshot1 = - kv.snapshot(682375462378L, System.currentTimeMillis()); - - // make some more modifications - kv.setCurrentKey(1); - kv.update("u1"); - kv.setCurrentKey(2); - kv.update("u2"); - kv.setCurrentKey(3); - kv.update("u3"); - - // draw another snapshot - KvStateSnapshot snapshot2 = - kv.snapshot(682375462379L, System.currentTimeMillis()); - - // validate the original state - assertEquals(3, kv.size()); - kv.setCurrentKey(1); - assertEquals("u1", kv.value()); - kv.setCurrentKey(2); - assertEquals("u2", kv.value()); - kv.setCurrentKey(3); - assertEquals("u3", kv.value()); - - // restore the first snapshot and validate it - KvState restored1 = snapshot1.restoreState(backend, - IntSerializer.INSTANCE, StringSerializer.INSTANCE, null, getClass().getClassLoader(), 1); - - assertEquals(2, restored1.size()); - restored1.setCurrentKey(1); - assertEquals("1", restored1.value()); - restored1.setCurrentKey(2); - assertEquals("2", restored1.value()); - - // restore the first snapshot and validate it - KvState restored2 = snapshot2.restoreState(backend, - IntSerializer.INSTANCE, StringSerializer.INSTANCE, null, getClass().getClassLoader(), 1); - - assertEquals(3, restored2.size()); - restored2.setCurrentKey(1); - assertEquals("u1", restored2.value()); - restored2.setCurrentKey(2); - assertEquals("u2", restored2.value()); - restored2.setCurrentKey(3); - assertEquals("u3", restored2.value()); - - snapshot1.discardState(); - assertFalse(isDirectoryEmpty(checkpointDir)); - - snapshot2.discardState(); - assertTrue(isDirectoryEmpty(checkpointDir)); - } - catch (Exception e) { - e.printStackTrace(); - fail(e.getMessage()); - } - finally { - deleteDirectorySilently(tempDir); - } - } - - @Test - public void testRestoreWithWrongSerializers() { - File tempDir = new File(ConfigConstants.DEFAULT_TASK_MANAGER_TMP_PATH, UUID.randomUUID().toString()); - try { - FsStateBackend backend = CommonTestUtils.createCopySerializable(new FsStateBackend(localFileUri(tempDir))); - backend.initializeForJob(new DummyEnvironment("test", 1, 0)); - - File checkpointDir = new File(backend.getCheckpointDirectory().toUri().getPath()); - - KvState kv = - backend.createKvState("a_0", "a", IntSerializer.INSTANCE, StringSerializer.INSTANCE, null); - - kv.setCurrentKey(1); - kv.update("1"); - kv.setCurrentKey(2); - kv.update("2"); - - KvStateSnapshot snapshot = - kv.snapshot(682375462378L, System.currentTimeMillis()); - - - @SuppressWarnings("unchecked") - TypeSerializer fakeIntSerializer = - (TypeSerializer) (TypeSerializer) FloatSerializer.INSTANCE; - - @SuppressWarnings("unchecked") - TypeSerializer fakeStringSerializer = - (TypeSerializer) (TypeSerializer) new ValueSerializer(StringValue.class); - - try { - snapshot.restoreState(backend, fakeIntSerializer, - StringSerializer.INSTANCE, null, getClass().getClassLoader(), 1); - fail("should recognize wrong serializers"); - } catch (IllegalArgumentException e) { - // expected - } catch (Exception e) { - fail("wrong exception"); - } - - try { - snapshot.restoreState(backend, IntSerializer.INSTANCE, - fakeStringSerializer, null, getClass().getClassLoader(), 1); - fail("should recognize wrong serializers"); - } catch (IllegalArgumentException e) { - // expected - } catch (Exception e) { - fail("wrong exception"); - } - - try { - snapshot.restoreState(backend, fakeIntSerializer, - fakeStringSerializer, null, getClass().getClassLoader(), 1); - fail("should recognize wrong serializers"); - } catch (IllegalArgumentException e) { - // expected - } catch (Exception e) { - fail("wrong exception"); - } - - snapshot.discardState(); - - assertTrue(isDirectoryEmpty(checkpointDir)); - } - catch (Exception e) { - e.printStackTrace(); - fail(e.getMessage()); - } - finally { - deleteDirectorySilently(tempDir); - } - } - - @Test - public void testCopyDefaultValue() { - File tempDir = new File(ConfigConstants.DEFAULT_TASK_MANAGER_TMP_PATH, UUID.randomUUID().toString()); - try { - FsStateBackend backend = CommonTestUtils.createCopySerializable(new FsStateBackend(localFileUri(tempDir))); - backend.initializeForJob(new DummyEnvironment("test", 1, 0)); - - KvState kv = - backend.createKvState("a_0", "a", IntSerializer.INSTANCE, IntValueSerializer.INSTANCE, new IntValue(-1)); - - kv.setCurrentKey(1); - IntValue default1 = kv.value(); - - kv.setCurrentKey(2); - IntValue default2 = kv.value(); - - assertNotNull(default1); - assertNotNull(default2); - assertEquals(default1, default2); - assertFalse(default1 == default2); - } - catch (Exception e) { - e.printStackTrace(); - fail(e.getMessage()); - } - finally { - deleteDirectorySilently(tempDir); - } - } - // ------------------------------------------------------------------------ // Utilities // ------------------------------------------------------------------------ @@ -437,6 +243,9 @@ private static void deleteDirectorySilently(File dir) { } private static boolean isDirectoryEmpty(File directory) { + if (!directory.exists()) { + return true; + } String[] nested = directory.list(); return nested == null || nested.length == 0; } @@ -447,15 +256,16 @@ private static String localFileUri(File path) { private static void validateBytesInStream(InputStream is, byte[] data) throws IOException { byte[] holder = new byte[data.length]; - int numBytesRead = is.read(holder); - - if (holder.length == 0) { - assertTrue("stream not empty", numBytesRead == 0 || numBytesRead == -1); - } else { - assertEquals("not enough data", holder.length, numBytesRead); + + int pos = 0; + int read; + while (pos < holder.length && (read = is.read(holder, pos, holder.length - pos)) != -1) { + pos += read; } - + + assertEquals("not enough data", holder.length, pos); assertEquals("too much data", -1, is.read()); assertArrayEquals("wrong data", data, holder); } + } diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/state/FsCheckpointStateOutputStreamTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/state/FsCheckpointStateOutputStreamTest.java index 66a727101553c..5964b720645ba 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/state/FsCheckpointStateOutputStreamTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/state/FsCheckpointStateOutputStreamTest.java @@ -48,7 +48,7 @@ public void testWrongParameters() { @Test public void testEmptyState() throws Exception { - StateBackend.CheckpointStateOutputStream stream = new FsStateBackend.FsCheckpointStateOutputStream( + AbstractStateBackend.CheckpointStateOutputStream stream = new FsStateBackend.FsCheckpointStateOutputStream( TEMP_DIR_PATH, FileSystem.getLocalFileSystem(), 1024, 512); StreamStateHandle handle = stream.closeAndGetHandle(); @@ -79,7 +79,7 @@ public void testZeroThreshold() throws Exception { } private void runTest(int numBytes, int bufferSize, int threshold, boolean expectFile) throws Exception { - StateBackend.CheckpointStateOutputStream stream = + AbstractStateBackend.CheckpointStateOutputStream stream = new FsStateBackend.FsCheckpointStateOutputStream( TEMP_DIR_PATH, FileSystem.getLocalFileSystem(), bufferSize, threshold); diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/state/MemoryStateBackendTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/state/MemoryStateBackendTest.java index 4b5aebd0c74cf..34354c1b0036e 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/state/MemoryStateBackendTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/state/MemoryStateBackendTest.java @@ -18,15 +18,7 @@ package org.apache.flink.runtime.state; -import org.apache.flink.api.common.typeutils.TypeSerializer; -import org.apache.flink.api.common.typeutils.base.FloatSerializer; -import org.apache.flink.api.common.typeutils.base.IntSerializer; -import org.apache.flink.api.common.typeutils.base.IntValueSerializer; -import org.apache.flink.api.common.typeutils.base.StringSerializer; -import org.apache.flink.api.java.typeutils.runtime.ValueSerializer; import org.apache.flink.runtime.state.memory.MemoryStateBackend; -import org.apache.flink.types.IntValue; -import org.apache.flink.types.StringValue; import org.junit.Test; import java.io.IOException; @@ -39,7 +31,15 @@ /** * Tests for the {@link org.apache.flink.runtime.state.memory.MemoryStateBackend}. */ -public class MemoryStateBackendTest { +public class MemoryStateBackendTest extends StateBackendTestBase { + + @Override + protected MemoryStateBackend getStateBackend() throws Exception { + return new MemoryStateBackend(); + } + + @Override + protected void cleanup() throws Exception { } @Test public void testSerializableState() { @@ -94,7 +94,7 @@ public void testStateStream() { state.put("hey there", 2); state.put("the crazy brown fox stumbles over a sentence that does not contain every letter", 77); - StateBackend.CheckpointStateOutputStream os = backend.createCheckpointStateOutputStream(1, 2); + AbstractStateBackend.CheckpointStateOutputStream os = backend.createCheckpointStateOutputStream(1, 2); ObjectOutputStream oos = new ObjectOutputStream(os); oos.writeObject(state); oos.flush(); @@ -122,7 +122,7 @@ public void testOversizedStateStream() { state.put("hey there", 2); state.put("the crazy brown fox stumbles over a sentence that does not contain every letter", 77); - StateBackend.CheckpointStateOutputStream os = backend.createCheckpointStateOutputStream(1, 2); + AbstractStateBackend.CheckpointStateOutputStream os = backend.createCheckpointStateOutputStream(1, 2); ObjectOutputStream oos = new ObjectOutputStream(os); try { @@ -140,164 +140,4 @@ public void testOversizedStateStream() { fail(e.getMessage()); } } - - @Test - public void testKeyValueState() { - try { - MemoryStateBackend backend = new MemoryStateBackend(); - - KvState kv = - backend.createKvState("s_0", "s", IntSerializer.INSTANCE, StringSerializer.INSTANCE, null); - - assertEquals(0, kv.size()); - - // some modifications to the state - kv.setCurrentKey(1); - assertNull(kv.value()); - kv.update("1"); - assertEquals(1, kv.size()); - kv.setCurrentKey(2); - assertNull(kv.value()); - kv.update("2"); - assertEquals(2, kv.size()); - kv.setCurrentKey(1); - assertEquals("1", kv.value()); - assertEquals(2, kv.size()); - - // draw a snapshot - KvStateSnapshot snapshot1 = - kv.snapshot(682375462378L, System.currentTimeMillis()); - - // make some more modifications - kv.setCurrentKey(1); - kv.update("u1"); - kv.setCurrentKey(2); - kv.update("u2"); - kv.setCurrentKey(3); - kv.update("u3"); - - // draw another snapshot - KvStateSnapshot snapshot2 = - kv.snapshot(682375462379L, System.currentTimeMillis()); - - // validate the original state - assertEquals(3, kv.size()); - kv.setCurrentKey(1); - assertEquals("u1", kv.value()); - kv.setCurrentKey(2); - assertEquals("u2", kv.value()); - kv.setCurrentKey(3); - assertEquals("u3", kv.value()); - - // restore the first snapshot and validate it - KvState restored1 = snapshot1.restoreState(backend, - IntSerializer.INSTANCE, StringSerializer.INSTANCE, null, getClass().getClassLoader(), 1); - - assertEquals(2, restored1.size()); - restored1.setCurrentKey(1); - assertEquals("1", restored1.value()); - restored1.setCurrentKey(2); - assertEquals("2", restored1.value()); - - // restore the first snapshot and validate it - KvState restored2 = snapshot2.restoreState(backend, - IntSerializer.INSTANCE, StringSerializer.INSTANCE, null, getClass().getClassLoader(), 1); - - assertEquals(3, restored2.size()); - restored2.setCurrentKey(1); - assertEquals("u1", restored2.value()); - restored2.setCurrentKey(2); - assertEquals("u2", restored2.value()); - restored2.setCurrentKey(3); - assertEquals("u3", restored2.value()); - } - catch (Exception e) { - e.printStackTrace(); - fail(e.getMessage()); - } - } - - @Test - public void testRestoreWithWrongSerializers() { - try { - MemoryStateBackend backend = new MemoryStateBackend(); - KvState kv = - backend.createKvState("s_0", "s", IntSerializer.INSTANCE, StringSerializer.INSTANCE, null); - - kv.setCurrentKey(1); - kv.update("1"); - kv.setCurrentKey(2); - kv.update("2"); - - KvStateSnapshot snapshot = - kv.snapshot(682375462378L, System.currentTimeMillis()); - - - @SuppressWarnings("unchecked") - TypeSerializer fakeIntSerializer = - (TypeSerializer) (TypeSerializer) FloatSerializer.INSTANCE; - - @SuppressWarnings("unchecked") - TypeSerializer fakeStringSerializer = - (TypeSerializer) (TypeSerializer) new ValueSerializer(StringValue.class); - - try { - snapshot.restoreState(backend, fakeIntSerializer, - StringSerializer.INSTANCE, null, getClass().getClassLoader(), 1); - fail("should recognize wrong serializers"); - } catch (IllegalArgumentException e) { - // expected - } catch (Exception e) { - fail("wrong exception"); - } - - try { - snapshot.restoreState(backend, IntSerializer.INSTANCE, - fakeStringSerializer, null, getClass().getClassLoader(), 1); - fail("should recognize wrong serializers"); - } catch (IllegalArgumentException e) { - // expected - } catch (Exception e) { - fail("wrong exception"); - } - - try { - snapshot.restoreState(backend, fakeIntSerializer, - fakeStringSerializer, null, getClass().getClassLoader(), 1); - fail("should recognize wrong serializers"); - } catch (IllegalArgumentException e) { - // expected - } catch (Exception e) { - fail("wrong exception"); - } - } - catch (Exception e) { - e.printStackTrace(); - fail(e.getMessage()); - } - } - - @Test - public void testCopyDefaultValue() { - try { - MemoryStateBackend backend = new MemoryStateBackend(); - KvState kv = - backend.createKvState("a_0", "a", IntSerializer.INSTANCE, IntValueSerializer.INSTANCE, new IntValue(-1)); - - kv.setCurrentKey(1); - IntValue default1 = kv.value(); - - kv.setCurrentKey(2); - IntValue default2 = kv.value(); - - assertNotNull(default1); - assertNotNull(default2); - assertEquals(default1, default2); - assertFalse(default1 == default2); - } - catch (Exception e) { - e.printStackTrace(); - fail(e.getMessage()); - } - } } diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/state/StateBackendTestBase.java b/flink-runtime/src/test/java/org/apache/flink/runtime/state/StateBackendTestBase.java new file mode 100644 index 0000000000000..82ab3b3b87fc2 --- /dev/null +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/state/StateBackendTestBase.java @@ -0,0 +1,494 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.runtime.state; + +import com.google.common.base.Joiner; +import org.apache.flink.api.common.functions.ReduceFunction; +import org.apache.flink.api.common.state.ListState; +import org.apache.flink.api.common.state.ListStateDescriptor; +import org.apache.flink.api.common.state.ReducingState; +import org.apache.flink.api.common.state.ReducingStateDescriptor; +import org.apache.flink.api.common.state.ValueState; +import org.apache.flink.api.common.state.ValueStateDescriptor; +import org.apache.flink.api.common.typeutils.TypeSerializer; +import org.apache.flink.api.common.typeutils.base.FloatSerializer; +import org.apache.flink.api.common.typeutils.base.IntSerializer; +import org.apache.flink.api.common.typeutils.base.IntValueSerializer; +import org.apache.flink.api.common.typeutils.base.StringSerializer; +import org.apache.flink.api.common.typeutils.base.VoidSerializer; +import org.apache.flink.runtime.operators.testutils.DummyEnvironment; +import org.apache.flink.types.IntValue; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import static org.junit.Assert.*; + +/** + * Generic tests for the partitioned state part of {@link AbstractStateBackend}. + */ +public abstract class StateBackendTestBase { + + protected B backend; + + protected abstract B getStateBackend() throws Exception; + + protected abstract void cleanup() throws Exception; + + @Before + public void setup() throws Exception { + this.backend = getStateBackend(); + } + + @After + public void teardown() throws Exception { + this.backend.dispose(); + cleanup(); + } + + @Test + public void testValueState() throws Exception { + + backend.initializeForJob(new DummyEnvironment("test", 1, 0), "test_op", IntSerializer.INSTANCE); + + ValueStateDescriptor kvId = new ValueStateDescriptor<>("id", null, StringSerializer.INSTANCE); + ValueState state = backend.getPartitionedState(null, VoidSerializer.INSTANCE, kvId); + + @SuppressWarnings("unchecked") + KvState, ValueStateDescriptor, B> kv = + (KvState, ValueStateDescriptor, B>) state; + + // some modifications to the state + kv.setCurrentKey(1); + assertNull(state.value()); + state.update("1"); + kv.setCurrentKey(2); + assertNull(state.value()); + state.update("2"); + kv.setCurrentKey(1); + assertEquals("1", state.value()); + + // draw a snapshot + KvStateSnapshot, ValueStateDescriptor, B> snapshot1 = + kv.snapshot(682375462378L, 2); + + // make some more modifications + kv.setCurrentKey(1); + state.update("u1"); + kv.setCurrentKey(2); + state.update("u2"); + kv.setCurrentKey(3); + state.update("u3"); + + // draw another snapshot + KvStateSnapshot, ValueStateDescriptor, B> snapshot2 = + kv.snapshot(682375462379L, 4); + + // validate the original state + kv.setCurrentKey(1); + assertEquals("u1", state.value()); + kv.setCurrentKey(2); + assertEquals("u2", state.value()); + kv.setCurrentKey(3); + assertEquals("u3", state.value()); + + kv.dispose(); + +// restore the first snapshot and validate it + KvState, ValueStateDescriptor, B> restored1 = snapshot1.restoreState( + backend, + IntSerializer.INSTANCE, + this.getClass().getClassLoader(), 10); + + @SuppressWarnings("unchecked") + ValueState restored1State = (ValueState) restored1; + + restored1.setCurrentKey(1); + assertEquals("1", restored1State.value()); + restored1.setCurrentKey(2); + assertEquals("2", restored1State.value()); + + restored1.dispose(); + + // restore the second snapshot and validate it + KvState, ValueStateDescriptor, B> restored2 = snapshot2.restoreState( + backend, + IntSerializer.INSTANCE, + this.getClass().getClassLoader(), 10); + + @SuppressWarnings("unchecked") + ValueState restored2State = (ValueState) restored2; + + restored2.setCurrentKey(1); + assertEquals("u1", restored2State.value()); + restored2.setCurrentKey(2); + assertEquals("u2", restored2State.value()); + restored2.setCurrentKey(3); + assertEquals("u3", restored2State.value()); + } + + @Test + @SuppressWarnings("unchecked,rawtypes") + public void testListState() { + try { + backend.initializeForJob(new DummyEnvironment("test", 1, 0), "test_op", IntSerializer.INSTANCE); + + ListStateDescriptor kvId = new ListStateDescriptor<>("id", StringSerializer.INSTANCE); + ListState state = backend.getPartitionedState(null, VoidSerializer.INSTANCE, kvId); + + @SuppressWarnings("unchecked") + KvState, ListStateDescriptor, B> kv = + (KvState, ListStateDescriptor, B>) state; + + Joiner joiner = Joiner.on(","); + // some modifications to the state + kv.setCurrentKey(1); + assertEquals("", joiner.join(state.get())); + state.add("1"); + kv.setCurrentKey(2); + assertEquals("", joiner.join(state.get())); + state.add("2"); + kv.setCurrentKey(1); + assertEquals("1", joiner.join(state.get())); + + // draw a snapshot + KvStateSnapshot, ListStateDescriptor, B> snapshot1 = + kv.snapshot(682375462378L, 2); + + // make some more modifications + kv.setCurrentKey(1); + state.add("u1"); + kv.setCurrentKey(2); + state.add("u2"); + kv.setCurrentKey(3); + state.add("u3"); + + // draw another snapshot + KvStateSnapshot, ListStateDescriptor, B> snapshot2 = + kv.snapshot(682375462379L, 4); + + // validate the original state + kv.setCurrentKey(1); + assertEquals("1,u1", joiner.join(state.get())); + kv.setCurrentKey(2); + assertEquals("2,u2", joiner.join(state.get())); + kv.setCurrentKey(3); + assertEquals("u3", joiner.join(state.get())); + + kv.dispose(); + + // restore the first snapshot and validate it + KvState, ListStateDescriptor, B> restored1 = snapshot1.restoreState( + backend, + IntSerializer.INSTANCE, + this.getClass().getClassLoader(), 10); + + @SuppressWarnings("unchecked") + ListState restored1State = (ListState) restored1; + + restored1.setCurrentKey(1); + assertEquals("1", joiner.join(restored1State.get())); + restored1.setCurrentKey(2); + assertEquals("2", joiner.join(restored1State.get())); + + restored1.dispose(); + + // restore the second snapshot and validate it + KvState, ListStateDescriptor, B> restored2 = snapshot2.restoreState( + backend, + IntSerializer.INSTANCE, + this.getClass().getClassLoader(), 20); + + @SuppressWarnings("unchecked") + ListState restored2State = (ListState) restored2; + + restored2.setCurrentKey(1); + assertEquals("1,u1", joiner.join(restored2State.get())); + restored2.setCurrentKey(2); + assertEquals("2,u2", joiner.join(restored2State.get())); + restored2.setCurrentKey(3); + assertEquals("u3", joiner.join(restored2State.get())); + } + catch (Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + @Test + @SuppressWarnings("unchecked,rawtypes") + public void testReducingState() { + try { + backend.initializeForJob(new DummyEnvironment("test", 1, 0), "test_op", IntSerializer.INSTANCE); + + ReducingStateDescriptor kvId = new ReducingStateDescriptor<>("id", + new ReduceFunction() { + private static final long serialVersionUID = 1L; + + @Override + public String reduce(String value1, String value2) throws Exception { + return value1 + "," + value2; + } + }, + StringSerializer.INSTANCE); + ReducingState state = backend.getPartitionedState(null, VoidSerializer.INSTANCE, kvId); + + @SuppressWarnings("unchecked") + KvState, ReducingStateDescriptor, B> kv = + (KvState, ReducingStateDescriptor, B>) state; + + Joiner joiner = Joiner.on(","); + // some modifications to the state + kv.setCurrentKey(1); + assertEquals(null, state.get()); + state.add("1"); + kv.setCurrentKey(2); + assertEquals(null, state.get()); + state.add("2"); + kv.setCurrentKey(1); + assertEquals("1", state.get()); + + // draw a snapshot + KvStateSnapshot, ReducingStateDescriptor, B> snapshot1 = + kv.snapshot(682375462378L, 2); + + // make some more modifications + kv.setCurrentKey(1); + state.add("u1"); + kv.setCurrentKey(2); + state.add("u2"); + kv.setCurrentKey(3); + state.add("u3"); + + // draw another snapshot + KvStateSnapshot, ReducingStateDescriptor, B> snapshot2 = + kv.snapshot(682375462379L, 4); + + // validate the original state + kv.setCurrentKey(1); + assertEquals("1,u1", state.get()); + kv.setCurrentKey(2); + assertEquals("2,u2", state.get()); + kv.setCurrentKey(3); + assertEquals("u3", state.get()); + + kv.dispose(); + + // restore the first snapshot and validate it + KvState, ReducingStateDescriptor, B> restored1 = snapshot1.restoreState( + backend, + IntSerializer.INSTANCE, + this.getClass().getClassLoader(), 10); + + @SuppressWarnings("unchecked") + ReducingState restored1State = (ReducingState) restored1; + + restored1.setCurrentKey(1); + assertEquals("1", restored1State.get()); + restored1.setCurrentKey(2); + assertEquals("2", restored1State.get()); + + restored1.dispose(); + + // restore the second snapshot and validate it + KvState, ReducingStateDescriptor, B> restored2 = snapshot2.restoreState( + backend, + IntSerializer.INSTANCE, + this.getClass().getClassLoader(), 20); + + @SuppressWarnings("unchecked") + ReducingState restored2State = (ReducingState) restored2; + + restored2.setCurrentKey(1); + assertEquals("1,u1", restored2State.get()); + restored2.setCurrentKey(2); + assertEquals("2,u2", restored2State.get()); + restored2.setCurrentKey(3); + assertEquals("u3", restored2State.get()); + } + catch (Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + + @Test + public void testValueStateRestoreWithWrongSerializers() { + try { + backend.initializeForJob(new DummyEnvironment("test", 1, 0), + "test_op", + IntSerializer.INSTANCE); + + ValueStateDescriptor kvId = new ValueStateDescriptor<>("id", + null, + StringSerializer.INSTANCE); + ValueState state = backend.getPartitionedState(null, + VoidSerializer.INSTANCE, + kvId); + + @SuppressWarnings("unchecked") + KvState, ValueStateDescriptor, B> kv = + (KvState, ValueStateDescriptor, B>) state; + + kv.setCurrentKey(1); + state.update("1"); + kv.setCurrentKey(2); + state.update("2"); + + KvStateSnapshot, ValueStateDescriptor, B> snapshot = + kv.snapshot(682375462378L, System.currentTimeMillis()); + + @SuppressWarnings("unchecked") + TypeSerializer fakeIntSerializer = + (TypeSerializer) (TypeSerializer) FloatSerializer.INSTANCE; + + try { + snapshot.restoreState(backend, fakeIntSerializer, getClass().getClassLoader(), 1); + fail("should recognize wrong serializers"); + } catch (IllegalArgumentException e) { + // expected + } catch (Exception e) { + fail("wrong exception"); + } + } + catch (Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + @Test + public void testListStateRestoreWithWrongSerializers() { + try { + backend.initializeForJob(new DummyEnvironment("test", 1, 0), "test_op", IntSerializer.INSTANCE); + + ListStateDescriptor kvId = new ListStateDescriptor<>("id", StringSerializer.INSTANCE); + ListState state = backend.getPartitionedState(null, VoidSerializer.INSTANCE, kvId); + + @SuppressWarnings("unchecked") + KvState, ListStateDescriptor, B> kv = + (KvState, ListStateDescriptor, B>) state; + + kv.setCurrentKey(1); + state.add("1"); + kv.setCurrentKey(2); + state.add("2"); + + KvStateSnapshot, ListStateDescriptor, B> snapshot = + kv.snapshot(682375462378L, System.currentTimeMillis()); + + kv.dispose(); + + @SuppressWarnings("unchecked") + TypeSerializer fakeIntSerializer = + (TypeSerializer) (TypeSerializer) FloatSerializer.INSTANCE; + + try { + snapshot.restoreState(backend, fakeIntSerializer, getClass().getClassLoader(), 1); + fail("should recognize wrong serializers"); + } catch (IllegalArgumentException e) { + // expected + } catch (Exception e) { + fail("wrong exception " + e); + } + } + catch (Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + @Test + public void testReducingStateRestoreWithWrongSerializers() { + try { + backend.initializeForJob(new DummyEnvironment("test", 1, 0), "test_op", IntSerializer.INSTANCE); + + ReducingStateDescriptor kvId = new ReducingStateDescriptor<>("id", + new ReduceFunction() { + @Override + public String reduce(String value1, String value2) throws Exception { + return value1 + "," + value2; + } + }, + StringSerializer.INSTANCE); + ReducingState state = backend.getPartitionedState(null, VoidSerializer.INSTANCE, kvId); + + @SuppressWarnings("unchecked") + KvState, ReducingStateDescriptor, B> kv = + (KvState, ReducingStateDescriptor, B>) state; + + kv.setCurrentKey(1); + state.add("1"); + kv.setCurrentKey(2); + state.add("2"); + + KvStateSnapshot, ReducingStateDescriptor, B> snapshot = + kv.snapshot(682375462378L, System.currentTimeMillis()); + + kv.dispose(); + + @SuppressWarnings("unchecked") + TypeSerializer fakeIntSerializer = + (TypeSerializer) (TypeSerializer) FloatSerializer.INSTANCE; + + try { + snapshot.restoreState(backend, fakeIntSerializer, getClass().getClassLoader(), 1); + fail("should recognize wrong serializers"); + } catch (IllegalArgumentException e) { + // expected + } catch (Exception e) { + fail("wrong exception " + e); + } + } + catch (Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + @Test + public void testCopyDefaultValue() { + try { + backend.initializeForJob(new DummyEnvironment("test", 1, 0), "test_op", IntSerializer.INSTANCE); + + ValueStateDescriptor kvId = new ValueStateDescriptor<>("id", new IntValue(-1), IntValueSerializer.INSTANCE); + ValueState state = backend.getPartitionedState(null, VoidSerializer.INSTANCE, kvId); + + @SuppressWarnings("unchecked") + KvState, ValueStateDescriptor, B> kv = + (KvState, ValueStateDescriptor, B>) state; + + kv.setCurrentKey(1); + IntValue default1 = state.value(); + + kv.setCurrentKey(2); + IntValue default2 = state.value(); + + assertNotNull(default1); + assertNotNull(default2); + assertEquals(default1, default2); + assertFalse(default1 == default2); + } + catch (Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } +} diff --git a/flink-streaming-connectors/flink-connector-filesystem/src/main/java/org/apache/flink/streaming/connectors/fs/RollingSink.java b/flink-streaming-connectors/flink-connector-filesystem/src/main/java/org/apache/flink/streaming/connectors/fs/RollingSink.java index 2112b28d22888..afae68f67a3b5 100644 --- a/flink-streaming-connectors/flink-connector-filesystem/src/main/java/org/apache/flink/streaming/connectors/fs/RollingSink.java +++ b/flink-streaming-connectors/flink-connector-filesystem/src/main/java/org/apache/flink/streaming/connectors/fs/RollingSink.java @@ -22,7 +22,7 @@ import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.typeutils.InputTypeConfigurable; import org.apache.flink.configuration.Configuration; -import org.apache.flink.streaming.api.checkpoint.CheckpointNotifier; +import org.apache.flink.runtime.state.CheckpointListener; import org.apache.flink.streaming.api.checkpoint.Checkpointed; import org.apache.flink.streaming.api.functions.sink.RichSinkFunction; import org.apache.hadoop.fs.FSDataOutputStream; @@ -119,7 +119,7 @@ * * @param Type of the elements emitted by this sink */ -public class RollingSink extends RichSinkFunction implements InputTypeConfigurable, Checkpointed, CheckpointNotifier { +public class RollingSink extends RichSinkFunction implements InputTypeConfigurable, Checkpointed, CheckpointListener { private static final long serialVersionUID = 1L; private static Logger LOG = LoggerFactory.getLogger(RollingSink.class); diff --git a/flink-streaming-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumerBase.java b/flink-streaming-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumerBase.java index 3c3658686a8aa..a51363701530a 100644 --- a/flink-streaming-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumerBase.java +++ b/flink-streaming-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumerBase.java @@ -20,7 +20,7 @@ import org.apache.commons.collections.map.LinkedMap; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.typeutils.ResultTypeQueryable; -import org.apache.flink.streaming.api.checkpoint.CheckpointNotifier; +import org.apache.flink.runtime.state.CheckpointListener; import org.apache.flink.streaming.api.checkpoint.CheckpointedAsynchronously; import org.apache.flink.streaming.api.functions.source.RichParallelSourceFunction; import org.apache.flink.streaming.connectors.kafka.internals.KafkaTopicPartition; @@ -39,7 +39,7 @@ public abstract class FlinkKafkaConsumerBase extends RichParallelSourceFunction - implements CheckpointNotifier, CheckpointedAsynchronously>, ResultTypeQueryable { + implements CheckpointListener, CheckpointedAsynchronously>, ResultTypeQueryable { // ------------------------------------------------------------------------ diff --git a/flink-streaming-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaConsumerTestBase.java b/flink-streaming-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaConsumerTestBase.java index 85921820022b8..cc96c277a590d 100644 --- a/flink-streaming-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaConsumerTestBase.java +++ b/flink-streaming-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaConsumerTestBase.java @@ -44,7 +44,7 @@ import org.apache.flink.configuration.Configuration; import org.apache.flink.runtime.client.JobExecutionException; import org.apache.flink.runtime.jobmanager.scheduler.NoResourceAvailableException; -import org.apache.flink.streaming.api.checkpoint.CheckpointNotifier; +import org.apache.flink.runtime.state.CheckpointListener; import org.apache.flink.streaming.api.checkpoint.Checkpointed; import org.apache.flink.streaming.api.datastream.DataStream; import org.apache.flink.streaming.api.datastream.DataStreamSource; @@ -1289,7 +1289,7 @@ private static void printTopic(String topicName, int elements,DeserializationSch public static class BrokerKillingMapper extends RichMapFunction - implements Checkpointed, CheckpointNotifier { + implements Checkpointed, CheckpointListener { private static final long serialVersionUID = 6334389850158707313L; diff --git a/flink-streaming-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/testutils/FailingIdentityMapper.java b/flink-streaming-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/testutils/FailingIdentityMapper.java index 5a8ffaaa7219e..2bd400c5d3575 100644 --- a/flink-streaming-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/testutils/FailingIdentityMapper.java +++ b/flink-streaming-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/testutils/FailingIdentityMapper.java @@ -20,14 +20,14 @@ import org.apache.flink.api.common.functions.RichMapFunction; import org.apache.flink.configuration.Configuration; -import org.apache.flink.streaming.api.checkpoint.CheckpointNotifier; +import org.apache.flink.runtime.state.CheckpointListener; import org.apache.flink.streaming.api.checkpoint.Checkpointed; import org.slf4j.Logger; import org.slf4j.LoggerFactory; public class FailingIdentityMapper extends RichMapFunction implements - Checkpointed, CheckpointNotifier, Runnable { + Checkpointed, CheckpointListener, Runnable { private static final Logger LOG = LoggerFactory.getLogger(FailingIdentityMapper.class); diff --git a/flink-streaming-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/testutils/MockRuntimeContext.java b/flink-streaming-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/testutils/MockRuntimeContext.java index 50c57abe1fa69..ee246bb839b77 100644 --- a/flink-streaming-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/testutils/MockRuntimeContext.java +++ b/flink-streaming-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/testutils/MockRuntimeContext.java @@ -26,7 +26,9 @@ import org.apache.flink.api.common.accumulators.LongCounter; import org.apache.flink.api.common.cache.DistributedCache; import org.apache.flink.api.common.functions.BroadcastVariableInitializer; -import org.apache.flink.api.common.state.OperatorState; +import org.apache.flink.api.common.state.State; +import org.apache.flink.api.common.state.StateDescriptor; +import org.apache.flink.api.common.state.ValueState; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.runtime.memory.MemoryManager; import org.apache.flink.runtime.operators.testutils.MockEnvironment; @@ -146,12 +148,17 @@ public DistributedCache getDistributedCache() { } @Override - public OperatorState getKeyValueState(String name, Class stateType, S defaultState) { + public ValueState getKeyValueState(String name, Class stateType, S defaultState) { throw new UnsupportedOperationException(); } @Override - public OperatorState getKeyValueState(String name, TypeInformation stateType, S defaultState) { + public ValueState getKeyValueState(String name, TypeInformation stateType, S defaultState) { + throw new UnsupportedOperationException(); + } + + @Override + public S getPartitionedState(StateDescriptor stateDescriptor) { throw new UnsupportedOperationException(); } } diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/ConnectedStreams.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/ConnectedStreams.java index 4074a1da8d84b..395b3293f1c4e 100644 --- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/ConnectedStreams.java +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/ConnectedStreams.java @@ -321,6 +321,21 @@ CoFlatMapFunction.class, false, true, getType1(), getType2(), outTypeInfo, environment.getParallelism()); + if (inputStream1 instanceof KeyedStream && inputStream2 instanceof KeyedStream) { + KeyedStream keyedInput1 = (KeyedStream) inputStream1; + KeyedStream keyedInput2 = (KeyedStream) inputStream2; + + TypeInformation keyType1 = keyedInput1.getKeyType(); + TypeInformation keyType2 = keyedInput2.getKeyType(); + if (!(keyType1.canEqual(keyType2) && keyType1.equals(keyType2))) { + throw new UnsupportedOperationException("Key types if input KeyedStreams " + + "don't match: " + keyType1 + " and " + keyType2 + "."); + } + + transform.setStateKeySelectors(keyedInput1.getKeySelector(), keyedInput2.getKeySelector()); + transform.setStateKeyType(keyType1); + } + @SuppressWarnings({ "unchecked", "rawtypes" }) SingleOutputStreamOperator returnStream = new SingleOutputStreamOperator(environment, transform); diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/environment/StreamExecutionEnvironment.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/environment/StreamExecutionEnvironment.java index cb5fce5ff83de..f4b31841e2970 100644 --- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/environment/StreamExecutionEnvironment.java +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/environment/StreamExecutionEnvironment.java @@ -26,6 +26,7 @@ import org.apache.flink.api.common.functions.InvalidTypesException; import org.apache.flink.api.common.io.FileInputFormat; import org.apache.flink.api.common.io.InputFormat; +import org.apache.flink.api.common.state.ValueState; import org.apache.flink.api.common.typeinfo.BasicTypeInfo; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.ClosureCleaner; @@ -62,7 +63,7 @@ import org.apache.flink.streaming.api.graph.StreamGraph; import org.apache.flink.streaming.api.graph.StreamGraphGenerator; import org.apache.flink.streaming.api.operators.StreamSource; -import org.apache.flink.runtime.state.StateBackend; +import org.apache.flink.runtime.state.AbstractStateBackend; import org.apache.flink.streaming.api.transformations.StreamTransformation; import org.apache.flink.types.StringValue; import org.apache.flink.util.SplittableIterator; @@ -124,7 +125,7 @@ public abstract class StreamExecutionEnvironment { protected boolean isChainingEnabled = true; /** The state backend used for storing k/v state and state snapshots */ - private StateBackend defaultStateBackend; + private AbstractStateBackend defaultStateBackend; /** The time characteristic used by the data streams */ private TimeCharacteristic timeCharacteristic = DEFAULT_TIME_CHARACTERISTIC; @@ -376,7 +377,7 @@ public CheckpointingMode getCheckpointingMode() { /** * Sets the state backend that describes how to store and checkpoint operator state. It defines in - * what form the key/value state ({@link org.apache.flink.api.common.state.OperatorState}, accessible + * what form the key/value state ({@link ValueState}, accessible * from operations on {@link org.apache.flink.streaming.api.datastream.KeyedStream}) is maintained * (heap, managed memory, externally), and where state snapshots/checkpoints are stored, both for * the key/value state, and for checkpointed functions (implementing the interface @@ -396,7 +397,7 @@ public CheckpointingMode getCheckpointingMode() { * * @see #getStateBackend() */ - public StreamExecutionEnvironment setStateBackend(StateBackend backend) { + public StreamExecutionEnvironment setStateBackend(AbstractStateBackend backend) { this.defaultStateBackend = requireNonNull(backend); return this; } @@ -405,9 +406,9 @@ public StreamExecutionEnvironment setStateBackend(StateBackend backend) { * Returns the state backend that defines how to store and checkpoint state. * @return The state backend that defines how to store and checkpoint state. * - * @see #setStateBackend(StateBackend) + * @see #setStateBackend(AbstractStateBackend) */ - public StateBackend getStateBackend() { + public AbstractStateBackend getStateBackend() { return defaultStateBackend; } diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/MessageAcknowledgingSourceBase.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/MessageAcknowledgingSourceBase.java index 43858845aab3d..e7da5f8e41da4 100644 --- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/MessageAcknowledgingSourceBase.java +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/MessageAcknowledgingSourceBase.java @@ -31,7 +31,7 @@ import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.api.java.typeutils.TypeExtractor; import org.apache.flink.configuration.Configuration; -import org.apache.flink.streaming.api.checkpoint.CheckpointNotifier; +import org.apache.flink.runtime.state.CheckpointListener; import org.apache.flink.streaming.api.checkpoint.Checkpointed; import org.apache.flink.runtime.state.SerializedCheckpointData; import org.slf4j.Logger; @@ -78,7 +78,7 @@ */ public abstract class MessageAcknowledgingSourceBase extends RichSourceFunction - implements Checkpointed, CheckpointNotifier { + implements Checkpointed, CheckpointListener { private static final long serialVersionUID = -8689291992192955579L; diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamConfig.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamConfig.java index 11bf84fb79d3b..7a07c79bcad49 100644 --- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamConfig.java +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamConfig.java @@ -31,7 +31,7 @@ import org.apache.flink.streaming.api.CheckpointingMode; import org.apache.flink.streaming.api.collector.selector.OutputSelectorWrapper; import org.apache.flink.streaming.api.operators.StreamOperator; -import org.apache.flink.runtime.state.StateBackend; +import org.apache.flink.runtime.state.AbstractStateBackend; import org.apache.flink.streaming.runtime.tasks.StreamTaskException; import org.apache.flink.util.InstantiationUtil; @@ -370,7 +370,7 @@ public Map getTransitiveChainedTaskConfigs(ClassLoader cl // State backend // ------------------------------------------------------------------------ - public void setStateBackend(StateBackend backend) { + public void setStateBackend(AbstractStateBackend backend) { try { InstantiationUtil.writeObjectToConfig(backend, this.config, STATE_BACKEND); } catch (Exception e) { @@ -378,7 +378,7 @@ public void setStateBackend(StateBackend backend) { } } - public StateBackend getStateBackend(ClassLoader cl) { + public AbstractStateBackend getStateBackend(ClassLoader cl) { try { return InstantiationUtil.readObjectFromConfig(this.config, STATE_BACKEND, cl); } catch (Exception e) { @@ -386,17 +386,17 @@ public StateBackend getStateBackend(ClassLoader cl) { } } - public void setStatePartitioner(KeySelector partitioner) { + public void setStatePartitioner(int input, KeySelector partitioner) { try { - InstantiationUtil.writeObjectToConfig(partitioner, this.config, STATE_PARTITIONER); + InstantiationUtil.writeObjectToConfig(partitioner, this.config, STATE_PARTITIONER + input); } catch (IOException e) { throw new StreamTaskException("Could not serialize state partitioner.", e); } } - public KeySelector getStatePartitioner(ClassLoader cl) { + public KeySelector getStatePartitioner(int input, ClassLoader cl) { try { - return InstantiationUtil.readObjectFromConfig(this.config, STATE_PARTITIONER, cl); + return InstantiationUtil.readObjectFromConfig(this.config, STATE_PARTITIONER + input, cl); } catch (Exception e) { throw new StreamTaskException("Could not instantiate state partitioner.", e); } diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamGraph.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamGraph.java index fa8c9d48459ae..ea85f05040bf5 100644 --- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamGraph.java +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamGraph.java @@ -47,7 +47,7 @@ import org.apache.flink.streaming.api.operators.StreamOperator; import org.apache.flink.streaming.api.operators.StreamSource; import org.apache.flink.streaming.api.operators.TwoInputStreamOperator; -import org.apache.flink.runtime.state.StateBackend; +import org.apache.flink.runtime.state.AbstractStateBackend; import org.apache.flink.streaming.runtime.partitioner.ForwardPartitioner; import org.apache.flink.streaming.runtime.partitioner.RebalancePartitioner; import org.apache.flink.streaming.runtime.partitioner.StreamPartitioner; @@ -85,7 +85,7 @@ public class StreamGraph extends StreamingPlan { protected Map vertexIDtoBrokerID; protected Map vertexIDtoLoopTimeout; - private StateBackend stateBackend; + private AbstractStateBackend stateBackend; private Set> iterationSourceSinkPairs; @@ -132,11 +132,11 @@ public void setChaining(boolean chaining) { this.chaining = chaining; } - public void setStateBackend(StateBackend backend) { + public void setStateBackend(AbstractStateBackend backend) { this.stateBackend = backend; } - public StateBackend getStateBackend() { + public AbstractStateBackend getStateBackend() { return this.stateBackend; } @@ -363,9 +363,16 @@ public void setParallelism(Integer vertexID, int parallelism) { } } - public void setKey(Integer vertexID, KeySelector keySelector, TypeSerializer keySerializer) { + public void setOneInputStateKey(Integer vertexID, KeySelector keySelector, TypeSerializer keySerializer) { StreamNode node = getStreamNode(vertexID); - node.setStatePartitioner(keySelector); + node.setStatePartitioner1(keySelector); + node.setStateKeySerializer(keySerializer); + } + + public void setTwoInputStateKey(Integer vertexID, KeySelector keySelector1, KeySelector keySelector2, TypeSerializer keySerializer) { + StreamNode node = getStreamNode(vertexID); + node.setStatePartitioner1(keySelector1); + node.setStatePartitioner2(keySelector2); node.setStateKeySerializer(keySerializer); } diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamGraphGenerator.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamGraphGenerator.java index 91c5e0fb81802..f200beda50f56 100644 --- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamGraphGenerator.java +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamGraphGenerator.java @@ -439,7 +439,7 @@ private Collection transformSink(SinkTransformation sink) { if (sink.getStateKeySelector() != null) { TypeSerializer keySerializer = sink.getStateKeyType().createSerializer(env.getConfig()); - streamGraph.setKey(sink.getId(), sink.getStateKeySelector(), keySerializer); + streamGraph.setOneInputStateKey(sink.getId(), sink.getStateKeySelector(), keySerializer); } return Collections.emptyList(); @@ -469,10 +469,7 @@ private Collection transformOnInputTransform(OneInputTransfor if (transform.getStateKeySelector() != null) { TypeSerializer keySerializer = transform.getStateKeyType().createSerializer(env.getConfig()); - streamGraph.setKey(transform.getId(), transform.getStateKeySelector(), keySerializer); - } - if (transform.getStateKeyType() != null) { - + streamGraph.setOneInputStateKey(transform.getId(), transform.getStateKeySelector(), keySerializer); } streamGraph.setParallelism(transform.getId(), transform.getParallelism()); @@ -509,6 +506,12 @@ private Collection transformTwoInputTransform(TwoInputT transform.getOutputType(), transform.getName()); + if (transform.getStateKeySelector1() != null) { + TypeSerializer keySerializer = transform.getStateKeyType().createSerializer(env.getConfig()); + streamGraph.setTwoInputStateKey(transform.getId(), transform.getStateKeySelector1(), transform.getStateKeySelector2(), keySerializer); + } + + streamGraph.setParallelism(transform.getId(), transform.getParallelism()); for (Integer inputId: inputIds1) { diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamNode.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamNode.java index 77b7cb46f6ada..0a612f3df4be5 100644 --- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamNode.java +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamNode.java @@ -49,7 +49,8 @@ public class StreamNode implements Serializable { private String operatorName; private Integer slotSharingID; private boolean isolatedSlot = false; - private KeySelector statePartitioner; + private KeySelector statePartitioner1; + private KeySelector statePartitioner2; private TypeSerializer stateKeySerializer; private transient StreamOperator operator; @@ -228,12 +229,20 @@ public String toString() { return operatorName + "-" + id; } - public KeySelector getStatePartitioner() { - return statePartitioner; + public KeySelector getStatePartitioner1() { + return statePartitioner1; } - public void setStatePartitioner(KeySelector statePartitioner) { - this.statePartitioner = statePartitioner; + public KeySelector getStatePartitioner2() { + return statePartitioner2; + } + + public void setStatePartitioner1(KeySelector statePartitioner) { + this.statePartitioner1 = statePartitioner; + } + + public void setStatePartitioner2(KeySelector statePartitioner) { + this.statePartitioner2 = statePartitioner; } public TypeSerializer getStateKeySerializer() { diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamingJobGraphGenerator.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamingJobGraphGenerator.java index 50c6a156ef359..6227801eeebca 100644 --- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamingJobGraphGenerator.java +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamingJobGraphGenerator.java @@ -327,7 +327,8 @@ private void setVertexConfig(Integer vertexID, StreamConfig config, // so we use that one if checkpointing is not enabled config.setCheckpointMode(CheckpointingMode.AT_LEAST_ONCE); } - config.setStatePartitioner(vertex.getStatePartitioner()); + config.setStatePartitioner(0, vertex.getStatePartitioner1()); + config.setStatePartitioner(1, vertex.getStatePartitioner2()); config.setStateKeySerializer(vertex.getStateKeySerializer()); Class vertexClass = vertex.getJobVertexClass(); diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/AbstractStreamOperator.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/AbstractStreamOperator.java index 3f1cfae68485e..f8f26b5e360f1 100644 --- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/AbstractStreamOperator.java +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/AbstractStreamOperator.java @@ -19,15 +19,14 @@ package org.apache.flink.streaming.api.operators; import org.apache.flink.api.common.ExecutionConfig; -import org.apache.flink.api.common.state.OperatorState; -import org.apache.flink.api.common.typeinfo.TypeInformation; +import org.apache.flink.api.common.state.State; +import org.apache.flink.api.common.state.StateDescriptor; import org.apache.flink.api.common.typeutils.TypeSerializer; +import org.apache.flink.api.common.typeutils.base.VoidSerializer; import org.apache.flink.api.java.functions.KeySelector; -import org.apache.flink.streaming.api.checkpoint.CheckpointNotifier; import org.apache.flink.streaming.api.graph.StreamConfig; -import org.apache.flink.runtime.state.KvState; import org.apache.flink.runtime.state.KvStateSnapshot; -import org.apache.flink.runtime.state.StateBackend; +import org.apache.flink.runtime.state.AbstractStateBackend; import org.apache.flink.streaming.runtime.operators.Triggerable; import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; import org.apache.flink.streaming.runtime.tasks.StreamTask; @@ -37,7 +36,6 @@ import org.slf4j.LoggerFactory; import java.util.HashMap; -import java.util.Map; /** * Base class for all stream operators. Operators that contain a user function should extend the class @@ -81,22 +79,16 @@ public abstract class AbstractStreamOperator /** The runtime context for UDFs */ private transient StreamingRuntimeContext runtimeContext; - + // ---------------- key/value state ------------------ - + /** key selector used to get the key for the state. Non-null only is the operator uses key/value state */ - private transient KeySelector stateKeySelector; - - private transient KvState[] keyValueStates; - - private transient HashMap> keyValueStatesByName; - - private transient TypeSerializer keySerializer; - - private transient HashMap> keyValueStateSnapshots; + private transient KeySelector stateKeySelector1; + private transient KeySelector stateKeySelector2; + + /** The state backend that stores the state and checkpoints for this task */ + private AbstractStateBackend stateBackend = null; - private long recoveryTimestamp; - // ------------------------------------------------------------------------ // Life Cycle // ------------------------------------------------------------------------ @@ -107,6 +99,19 @@ public void setup(StreamTask containingTask, StreamConfig config, Output keySerializer = config.getStateKeySerializer(getUserCodeClassloader()); + // if the keySerializer is null we still need to create the state backend + // for the non-partitioned state features it provides, such as the state output streams + String operatorIdentifier = getClass().getSimpleName() + "_" + config.getVertexID() + "_" + runtimeContext.getIndexOfThisSubtask(); + stateBackend = container.createStateBackend(operatorIdentifier, keySerializer); + } catch (Exception e) { + throw new RuntimeException("Could not initialize state backend. ", e); + } } /** @@ -144,9 +149,12 @@ public void close() throws Exception {} */ @Override public void dispose() { - if (keyValueStates != null) { - for (KvState state : keyValueStates) { - state.dispose(); + if (stateBackend != null) { + try { + stateBackend.close(); + stateBackend.dispose(); + } catch (Exception e) { + throw new RuntimeException("Error while closing/disposing state backend.", e); } } } @@ -160,37 +168,33 @@ public StreamTaskState snapshotOperatorState(long checkpointId, long timestamp) // here, we deal with key/value state snapshots StreamTaskState state = new StreamTaskState(); - if (keyValueStates != null) { - HashMap> snapshots = new HashMap<>(keyValueStatesByName.size()); - - for (Map.Entry> entry : keyValueStatesByName.entrySet()) { - KvStateSnapshot snapshot = entry.getValue().snapshot(checkpointId, timestamp); - snapshots.put(entry.getKey(), snapshot); + + if (stateBackend != null) { + HashMap> partitionedSnapshots = + stateBackend.snapshotPartitionedState(checkpointId, timestamp); + if (partitionedSnapshots != null) { + state.setKvStates(partitionedSnapshots); } - - state.setKvStates(snapshots); } - + + return state; } @Override + @SuppressWarnings("rawtypes,unchecked") public void restoreState(StreamTaskState state, long recoveryTimestamp) throws Exception { // restore the key/value state. the actual restore happens lazily, when the function requests // the state again, because the restore method needs information provided by the user function - keyValueStateSnapshots = state.getKvStates(); - this.recoveryTimestamp = recoveryTimestamp; + if (stateBackend != null) { + stateBackend.injectKeyValueStateSnapshots((HashMap)state.getKvStates(), recoveryTimestamp); + } } @Override public void notifyOfCompletedCheckpoint(long checkpointId) throws Exception { - // We check whether the KvStates require notifications - if (keyValueStates != null) { - for (KvState kvstate : keyValueStates) { - if (kvstate instanceof CheckpointNotifier) { - ((CheckpointNotifier) kvstate).notifyCheckpointComplete(checkpointId); - } - } + if (stateBackend != null) { + stateBackend.notifyOfCompletedCheckpoint(checkpointId); } } @@ -229,8 +233,8 @@ public StreamingRuntimeContext getRuntimeContext() { return runtimeContext; } - public StateBackend getStateBackend() { - return container.getStateBackend(); + public AbstractStateBackend getStateBackend() { + return stateBackend; } /** @@ -245,122 +249,50 @@ protected void registerTimer(long time, Triggerable target) { } /** - * Creates a key/value state handle, using the state backend configured for this task. - * - * @param stateType The type information for the state type, used for managed memory and state snapshots. - * @param defaultValue The default value that the state should return for keys that currently have - * no value associated with them - * - * @param The type of the state value. - * - * @return The key/value state for this operator. - * + * Creates a partitioned state handle, using the state backend configured for this task. + * * @throws IllegalStateException Thrown, if the key/value state was already initialized. * @throws Exception Thrown, if the state backend cannot create the key/value state. */ - protected OperatorState createKeyValueState( - String name, TypeInformation stateType, V defaultValue) throws Exception - { - return createKeyValueState(name, stateType.createSerializer(getExecutionConfig()), defaultValue); + protected S getPartitionedState(StateDescriptor stateDescriptor) throws Exception { + return getStateBackend().getPartitionedState(null, VoidSerializer.INSTANCE, stateDescriptor); } - + /** - * Creates a key/value state handle, using the state backend configured for this task. - * - * @param valueSerializer The type serializer for the state type, used for managed memory and state snapshots. - * @param defaultValue The default value that the state should return for keys that currently have - * no value associated with them - * - * @param The type of the state key. - * @param The type of the state value. - * @param The type of the state backend that creates the key/value state. - * - * @return The key/value state for this operator. - * + * Creates a partitioned state handle, using the state backend configured for this task. + * * @throws IllegalStateException Thrown, if the key/value state was already initialized. * @throws Exception Thrown, if the state backend cannot create the key/value state. */ @SuppressWarnings("unchecked") - protected > OperatorState createKeyValueState( - String name, TypeSerializer valueSerializer, V defaultValue) throws Exception - { - if (name == null || name.isEmpty()) { - throw new IllegalArgumentException(); - } - if (keyValueStatesByName != null && keyValueStatesByName.containsKey(name)) { - throw new IllegalStateException("The key/value state has already been created"); - } - - TypeSerializer keySerializer; - - // first time state access, make sure we load the state partitioner - if (stateKeySelector == null) { - stateKeySelector = config.getStatePartitioner(getUserCodeClassloader()); - if (stateKeySelector == null) { - throw new UnsupportedOperationException("The function or operator is not executed " + - "on a KeyedStream and can hence not access the key/value state"); - } - - keySerializer = config.getStateKeySerializer(getUserCodeClassloader()); - if (keySerializer == null) { - throw new Exception("State key serializer has not been configured in the config."); - } - this.keySerializer = keySerializer; - } - else if (this.keySerializer != null) { - keySerializer = (TypeSerializer) this.keySerializer; - } - else { - // should never happen, this is merely a safeguard - throw new RuntimeException(); - } - - Backend stateBackend = (Backend) container.getStateBackend(); - - KvState kvstate = null; - - // check whether we restore the key/value state from a snapshot, or create a new blank one - if (keyValueStateSnapshots != null) { - KvStateSnapshot snapshot = (KvStateSnapshot) keyValueStateSnapshots.remove(name); + protected S getPartitionedState(N namespace, TypeSerializer namespaceSerializer, StateDescriptor stateDescriptor) throws Exception { + return getStateBackend().getPartitionedState(namespace, (TypeSerializer) namespaceSerializer, + stateDescriptor); + } - if (snapshot != null) { - kvstate = snapshot.restoreState( - stateBackend, keySerializer, valueSerializer, defaultValue, getUserCodeClassloader(), recoveryTimestamp); - } - } - - if (kvstate == null) { - // create unique state id from operator id + state name - String stateId = name + "_" + getOperatorConfig().getVertexID(); - // create a new blank key/value state - kvstate = stateBackend.createKvState(stateId ,name , keySerializer, valueSerializer, defaultValue); - } - if (keyValueStatesByName == null) { - keyValueStatesByName = new HashMap<>(); + @Override + @SuppressWarnings({"unchecked", "rawtypes"}) + public void setKeyContextElement1(StreamRecord record) throws Exception { + if (stateKeySelector1 != null) { + Object key = ((KeySelector) stateKeySelector1).getKey(record.getValue()); + getStateBackend().setCurrentKey(key); } - keyValueStatesByName.put(name, kvstate); - keyValueStates = keyValueStatesByName.values().toArray(new KvState[keyValueStatesByName.size()]); - return kvstate; } - + @Override @SuppressWarnings({"unchecked", "rawtypes"}) - public void setKeyContextElement(StreamRecord record) throws Exception { - if (stateKeySelector != null && keyValueStates != null) { - KeySelector selector = stateKeySelector; - for (KvState kv : keyValueStates) { - kv.setCurrentKey(selector.getKey(record.getValue())); - } + public void setKeyContextElement2(StreamRecord record) throws Exception { + if (stateKeySelector2 != null) { + Object key = ((KeySelector) stateKeySelector2).getKey(record.getValue()); + getStateBackend().setCurrentKey(key); } } @SuppressWarnings({"unchecked", "rawtypes"}) public void setKeyContext(Object key) { - if (keyValueStates != null) { - for (KvState kv : keyValueStates) { - kv.setCurrentKey(key); - } + if (stateKeySelector1 != null) { + stateBackend.setCurrentKey(key); } } diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/AbstractUdfStreamOperator.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/AbstractUdfStreamOperator.java index c20544565ad13..37dd6ab493408 100644 --- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/AbstractUdfStreamOperator.java +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/AbstractUdfStreamOperator.java @@ -26,10 +26,10 @@ import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.configuration.Configuration; import org.apache.flink.runtime.state.StateHandle; -import org.apache.flink.streaming.api.checkpoint.CheckpointNotifier; +import org.apache.flink.runtime.state.CheckpointListener; import org.apache.flink.streaming.api.checkpoint.Checkpointed; import org.apache.flink.streaming.api.graph.StreamConfig; -import org.apache.flink.runtime.state.StateBackend; +import org.apache.flink.runtime.state.AbstractStateBackend; import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; import org.apache.flink.streaming.runtime.tasks.StreamTask; import org.apache.flink.streaming.runtime.tasks.StreamTaskState; @@ -98,6 +98,7 @@ public void close() throws Exception { @Override public void dispose() { + super.dispose(); if (!functionsClosed) { functionsClosed = true; try { @@ -131,7 +132,7 @@ public StreamTaskState snapshotOperatorState(long checkpointId, long timestamp) if (udfState != null) { try { - StateBackend stateBackend = getStateBackend(); + AbstractStateBackend stateBackend = getStateBackend(); StateHandle handle = stateBackend.checkpointStateSerializable(udfState, checkpointId, timestamp); state.setFunctionState(handle); @@ -172,8 +173,8 @@ public void restoreState(StreamTaskState state, long recoveryTimestamp) throws E public void notifyOfCompletedCheckpoint(long checkpointId) throws Exception { super.notifyOfCompletedCheckpoint(checkpointId); - if (userFunction instanceof CheckpointNotifier) { - ((CheckpointNotifier) userFunction).notifyCheckpointComplete(checkpointId); + if (userFunction instanceof CheckpointListener) { + ((CheckpointListener) userFunction).notifyCheckpointComplete(checkpointId); } } diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/StreamGroupedFold.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/StreamGroupedFold.java index c383935379ba9..e627ec8ecdc6e 100644 --- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/StreamGroupedFold.java +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/StreamGroupedFold.java @@ -23,7 +23,8 @@ import org.apache.flink.api.common.ExecutionConfig; import org.apache.flink.api.common.functions.FoldFunction; -import org.apache.flink.api.common.state.OperatorState; +import org.apache.flink.api.common.state.ValueState; +import org.apache.flink.api.common.state.ValueStateDescriptor; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.common.typeutils.TypeSerializer; import org.apache.flink.core.memory.DataInputViewStreamWrapper; @@ -40,7 +41,7 @@ public class StreamGroupedFold private static final String STATE_NAME = "_op_state"; // Grouped values - private transient OperatorState values; + private transient ValueState values; private transient OUT initialValue; @@ -66,7 +67,8 @@ public void open() throws Exception { ByteArrayInputStream bais = new ByteArrayInputStream(serializedInitialValue); DataInputViewStreamWrapper in = new DataInputViewStreamWrapper(bais); initialValue = outTypeSerializer.deserialize(in); - values = createKeyValueState(STATE_NAME, outTypeSerializer, null); + ValueStateDescriptor stateId = new ValueStateDescriptor<>(STATE_NAME, null, outTypeSerializer); + values = getPartitionedState(stateId); } @Override diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/StreamGroupedReduce.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/StreamGroupedReduce.java index ae15e92b9d482..c0545634d8383 100644 --- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/StreamGroupedReduce.java +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/StreamGroupedReduce.java @@ -18,7 +18,8 @@ package org.apache.flink.streaming.api.operators; import org.apache.flink.api.common.functions.ReduceFunction; -import org.apache.flink.api.common.state.OperatorState; +import org.apache.flink.api.common.state.ValueState; +import org.apache.flink.api.common.state.ValueStateDescriptor; import org.apache.flink.api.common.typeutils.TypeSerializer; import org.apache.flink.streaming.api.watermark.Watermark; import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; @@ -30,7 +31,7 @@ public class StreamGroupedReduce extends AbstractUdfStreamOperator values; + private transient ValueState values; private TypeSerializer serializer; @@ -43,7 +44,8 @@ public StreamGroupedReduce(ReduceFunction reducer, TypeSerializer serial @Override public void open() throws Exception { super.open(); - values = createKeyValueState(STATE_NAME, serializer, null); + ValueStateDescriptor stateId = new ValueStateDescriptor<>(STATE_NAME, null, serializer); + values = getPartitionedState(stateId); } @Override diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/StreamOperator.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/StreamOperator.java index 1ef3298316f5f..5c8267352492b 100644 --- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/StreamOperator.java +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/StreamOperator.java @@ -134,8 +134,10 @@ public interface StreamOperator extends Serializable { // miscellaneous // ------------------------------------------------------------------------ - void setKeyContextElement(StreamRecord record) throws Exception; - + void setKeyContextElement1(StreamRecord record) throws Exception; + + void setKeyContextElement2(StreamRecord record) throws Exception; + /** * An operator can return true here to disable copying of its input elements. This overrides * the object-reuse setting on the {@link org.apache.flink.api.common.ExecutionConfig} diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/StreamingRuntimeContext.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/StreamingRuntimeContext.java index 46f2fef100958..dda92bcbfab3a 100644 --- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/StreamingRuntimeContext.java +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/StreamingRuntimeContext.java @@ -21,7 +21,10 @@ import org.apache.flink.api.common.accumulators.Accumulator; import org.apache.flink.api.common.functions.BroadcastVariableInitializer; import org.apache.flink.api.common.functions.util.AbstractRuntimeUDFContext; -import org.apache.flink.api.common.state.OperatorState; +import org.apache.flink.api.common.state.ValueState; +import org.apache.flink.api.common.state.State; +import org.apache.flink.api.common.state.StateDescriptor; +import org.apache.flink.api.common.state.ValueStateDescriptor; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.typeutils.TypeExtractor; import org.apache.flink.runtime.execution.Environment; @@ -30,7 +33,6 @@ import org.apache.flink.streaming.api.graph.StreamConfig; import org.apache.flink.streaming.runtime.operators.Triggerable; -import java.util.HashMap; import java.util.List; import java.util.Map; @@ -47,17 +49,9 @@ public class StreamingRuntimeContext extends AbstractRuntimeUDFContext { /** The task environment running the operator */ private final Environment taskEnvironment; - - /** The key/value state, if the user-function requests it */ - private HashMap> keyValueStates; - - /** Type of the values stored in the state, to make sure repeated requests of the state are consistent */ - private HashMap> stateTypeInfos; - /** Stream configuration object. */ private final StreamConfig streamConfig; - public StreamingRuntimeContext(AbstractStreamOperator operator, Environment env, Map> accumulators) { super(env.getTaskInfo(), @@ -112,7 +106,17 @@ public C getBroadcastVariableWithInitializer(String name, BroadcastVariab // ------------------------------------------------------------------------ @Override - public OperatorState getKeyValueState(String name, Class stateType, S defaultState) { + public S getPartitionedState(StateDescriptor stateDescriptor) { + try { + return operator.getPartitionedState(stateDescriptor); + } catch (Exception e) { + throw new RuntimeException("Error while getting state.", e); + } + } + + @Override + @Deprecated + public ValueState getKeyValueState(String name, Class stateType, S defaultState) { requireNonNull(stateType, "The state type class must not be null"); TypeInformation typeInfo; @@ -120,62 +124,22 @@ public OperatorState getKeyValueState(String name, Class stateType, S typeInfo = TypeExtractor.getForClass(stateType); } catch (Exception e) { - throw new RuntimeException("Cannot analyze type '" + stateType.getName() + + throw new RuntimeException("Cannot analyze type '" + stateType.getName() + "' from the class alone, due to generic type parameters. " + "Please specify the TypeInformation directly.", e); } - + return getKeyValueState(name, typeInfo, defaultState); } @Override - public OperatorState getKeyValueState(String name, TypeInformation stateType, S defaultState) { + @Deprecated + public ValueState getKeyValueState(String name, TypeInformation stateType, S defaultState) { requireNonNull(name, "The name of the state must not be null"); requireNonNull(stateType, "The state type information must not be null"); - - OperatorState previousState; - - // check if this is a repeated call to access the state - if (this.stateTypeInfos != null && this.keyValueStates != null && - (previousState = this.keyValueStates.get(name)) != null) { - - // repeated call - TypeInformation previousType; - if (stateType.equals((previousType = this.stateTypeInfos.get(name)))) { - // valid case, same type requested again - @SuppressWarnings("unchecked") - OperatorState previous = (OperatorState) previousState; - return previous; - } - else { - // invalid case, different type requested this time - throw new IllegalStateException("Cannot initialize key/value state for type " + stateType + - " ; The key/value state has already been created and initialized for a different type: " + - previousType); - } - } - else { - // first time access to the key/value state - if (this.stateTypeInfos == null) { - this.stateTypeInfos = new HashMap<>(); - } - if (this.keyValueStates == null) { - this.keyValueStates = new HashMap<>(); - } - - try { - OperatorState state = operator.createKeyValueState(name, stateType, defaultState); - this.keyValueStates.put(name, state); - this.stateTypeInfos.put(name, stateType); - return state; - } - catch (RuntimeException e) { - throw e; - } - catch (Exception e) { - throw new RuntimeException("Cannot initialize the key/value state", e); - } - } + + ValueStateDescriptor stateDesc = new ValueStateDescriptor<>(name, defaultState, stateType.createSerializer(getExecutionConfig())); + return getPartitionedState(stateDesc); } // ------------------ expose (read only) relevant information from the stream config -------- // diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/transformations/TwoInputTransformation.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/transformations/TwoInputTransformation.java index 30f0733af1de9..b065df6e8296f 100644 --- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/transformations/TwoInputTransformation.java +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/transformations/TwoInputTransformation.java @@ -19,6 +19,7 @@ import com.google.common.collect.Lists; import org.apache.flink.api.common.typeinfo.TypeInformation; +import org.apache.flink.api.java.functions.KeySelector; import org.apache.flink.streaming.api.operators.ChainingStrategy; import org.apache.flink.streaming.api.operators.TwoInputStreamOperator; @@ -41,6 +42,12 @@ public class TwoInputTransformation extends StreamTransformation< private final TwoInputStreamOperator operator; + private KeySelector stateKeySelector1; + + private KeySelector stateKeySelector2; + + private TypeInformation stateKeyType; + /** * Creates a new {@code TwoInputTransformation} from the given inputs and operator. * @@ -99,6 +106,46 @@ public TwoInputStreamOperator getOperator() { return operator; } + /** + * Sets the {@link KeySelector KeySelectors} that must be used for partitioning keyed state of + * this transformation. + * + * @param stateKeySelector1 The {@code KeySelector} to set for the first input + * @param stateKeySelector2 The {@code KeySelector} to set for the first input + */ + public void setStateKeySelectors(KeySelector stateKeySelector1, KeySelector stateKeySelector2) { + this.stateKeySelector1 = stateKeySelector1; + this.stateKeySelector2 = stateKeySelector2; + } + + /** + * Returns the {@code KeySelector} that must be used for partitioning keyed state in this + * Operation for the first input. + * + * @see #setStateKeySelectors + */ + public KeySelector getStateKeySelector1() { + return stateKeySelector1; + } + + /** + * Returns the {@code KeySelector} that must be used for partitioning keyed state in this + * Operation for the second input. + * + * @see #setStateKeySelectors + */ + public KeySelector getStateKeySelector2() { + return stateKeySelector2; + } + + public void setStateKeyType(TypeInformation stateKeyType) { + this.stateKeyType = stateKeyType; + } + + public TypeInformation getStateKeyType() { + return stateKeyType; + } + @Override public Collection> getTransitivePredecessors() { List> result = Lists.newArrayList(); diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/triggers/ContinuousEventTimeTrigger.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/triggers/ContinuousEventTimeTrigger.java index 0454e85243684..b653be38f4aed 100644 --- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/triggers/ContinuousEventTimeTrigger.java +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/triggers/ContinuousEventTimeTrigger.java @@ -18,7 +18,7 @@ package org.apache.flink.streaming.api.windowing.triggers; import com.google.common.annotations.VisibleForTesting; -import org.apache.flink.api.common.state.OperatorState; +import org.apache.flink.api.common.state.ValueState; import org.apache.flink.streaming.api.windowing.time.Time; import org.apache.flink.streaming.api.windowing.windows.Window; @@ -42,7 +42,7 @@ private ContinuousEventTimeTrigger(long interval) { @Override public TriggerResult onElement(Object element, long timestamp, W window, TriggerContext ctx) throws Exception { - OperatorState first = ctx.getKeyValueState("first", true); + ValueState first = ctx.getKeyValueState("first", true); if (first.value()) { long start = timestamp - (timestamp % interval); diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/triggers/ContinuousProcessingTimeTrigger.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/triggers/ContinuousProcessingTimeTrigger.java index 357639494599f..7f3e7ec0ec17c 100644 --- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/triggers/ContinuousProcessingTimeTrigger.java +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/triggers/ContinuousProcessingTimeTrigger.java @@ -18,7 +18,7 @@ package org.apache.flink.streaming.api.windowing.triggers; import com.google.common.annotations.VisibleForTesting; -import org.apache.flink.api.common.state.OperatorState; +import org.apache.flink.api.common.state.ValueState; import org.apache.flink.streaming.api.windowing.time.Time; import org.apache.flink.streaming.api.windowing.windows.Window; @@ -41,7 +41,7 @@ private ContinuousProcessingTimeTrigger(long interval) { public TriggerResult onElement(Object element, long timestamp, W window, TriggerContext ctx) throws Exception { long currentTime = System.currentTimeMillis(); - OperatorState fireState = ctx.getKeyValueState("fire-timestamp", 0L); + ValueState fireState = ctx.getKeyValueState("fire-timestamp", 0L); long nextFireTimestamp = fireState.value(); if (nextFireTimestamp == 0) { @@ -70,7 +70,7 @@ public TriggerResult onEventTime(long time, W window, TriggerContext ctx) throws @Override public TriggerResult onProcessingTime(long time, W window, TriggerContext ctx) throws Exception { - OperatorState fireState = ctx.getKeyValueState("fire-timestamp", 0L); + ValueState fireState = ctx.getKeyValueState("fire-timestamp", 0L); long nextFireTimestamp = fireState.value(); // only fire if an element didn't already fire diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/triggers/CountTrigger.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/triggers/CountTrigger.java index efb62d7690f38..d101fe1e96dca 100644 --- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/triggers/CountTrigger.java +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/triggers/CountTrigger.java @@ -17,7 +17,7 @@ */ package org.apache.flink.streaming.api.windowing.triggers; -import org.apache.flink.api.common.state.OperatorState; +import org.apache.flink.api.common.state.ValueState; import org.apache.flink.streaming.api.windowing.windows.Window; import java.io.IOException; @@ -38,7 +38,7 @@ private CountTrigger(long maxCount) { @Override public TriggerResult onElement(Object element, long timestamp, W window, TriggerContext ctx) throws IOException { - OperatorState count = ctx.getKeyValueState("count", 0L); + ValueState count = ctx.getKeyValueState("count", 0L); long currentCount = count.value() + 1; count.update(currentCount); if (currentCount >= maxCount) { diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/triggers/DeltaTrigger.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/triggers/DeltaTrigger.java index d791d286ebce2..37c8a45c39bb8 100644 --- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/triggers/DeltaTrigger.java +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/triggers/DeltaTrigger.java @@ -17,7 +17,7 @@ */ package org.apache.flink.streaming.api.windowing.triggers; -import org.apache.flink.api.common.state.OperatorState; +import org.apache.flink.api.common.state.ValueState; import org.apache.flink.streaming.api.functions.windowing.delta.DeltaFunction; import org.apache.flink.streaming.api.windowing.windows.Window; @@ -46,7 +46,7 @@ private DeltaTrigger(double threshold, DeltaFunction deltaFunction) { @Override public TriggerResult onElement(T element, long timestamp, W window, TriggerContext ctx) throws Exception { - OperatorState lastElementState = ctx.getKeyValueState("last-element", null); + ValueState lastElementState = ctx.getKeyValueState("last-element", null); if (lastElementState.value() == null) { lastElementState.update(element); return TriggerResult.CONTINUE; diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/triggers/Trigger.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/triggers/Trigger.java index ee6a2791e96a8..56f133a60e465 100644 --- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/triggers/Trigger.java +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/triggers/Trigger.java @@ -17,7 +17,7 @@ */ package org.apache.flink.streaming.api.windowing.triggers; -import org.apache.flink.api.common.state.OperatorState; +import org.apache.flink.api.common.state.ValueState; import org.apache.flink.streaming.api.windowing.windows.Window; import java.io.Serializable; @@ -149,13 +149,13 @@ interface TriggerContext { void registerEventTimeTimer(long time); /** - * Retrieves an {@link OperatorState} object that can be used to interact with + * Retrieves an {@link ValueState} object that can be used to interact with * fault-tolerant state that is scoped to the window and key of the current * trigger invocation. * * @param name A unique key for the state. * @param defaultState The default value of the state. */ - OperatorState getKeyValueState(final String name, final S defaultState); + ValueState getKeyValueState(final String name, final S defaultState); } } diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/StreamInputProcessor.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/StreamInputProcessor.java index e131cda5bee5c..9dacc8dd29fa7 100644 --- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/StreamInputProcessor.java +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/StreamInputProcessor.java @@ -162,7 +162,7 @@ public boolean processInput(OneInputStreamOperator streamOperator, final // now we can do the actual processing StreamRecord record = recordOrWatermark.asRecord(); synchronized (lock) { - streamOperator.setKeyContextElement(record); + streamOperator.setKeyContextElement1(record); streamOperator.processElement(record); } return true; diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/StreamTwoInputProcessor.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/StreamTwoInputProcessor.java index 882037e30dae6..f639b4af3db68 100644 --- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/StreamTwoInputProcessor.java +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/StreamTwoInputProcessor.java @@ -186,6 +186,7 @@ public boolean processInput(TwoInputStreamOperator streamOperator, } else { synchronized (lock) { + streamOperator.setKeyContextElement1(recordOrWatermark.asRecord()); streamOperator.processElement1(recordOrWatermark.asRecord()); } return true; @@ -200,6 +201,7 @@ public boolean processInput(TwoInputStreamOperator streamOperator, } else { synchronized (lock) { + streamOperator.setKeyContextElement2(recordOrWatermark.asRecord()); streamOperator.processElement2(recordOrWatermark.asRecord()); } return true; diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/AbstractAlignedProcessingTimeWindowOperator.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/AbstractAlignedProcessingTimeWindowOperator.java index 677a7dd2db4d5..1be22ece493ce 100644 --- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/AbstractAlignedProcessingTimeWindowOperator.java +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/AbstractAlignedProcessingTimeWindowOperator.java @@ -29,7 +29,7 @@ import org.apache.flink.streaming.api.operators.AbstractUdfStreamOperator; import org.apache.flink.streaming.api.operators.OneInputStreamOperator; import org.apache.flink.streaming.api.operators.TimestampedCollector; -import org.apache.flink.runtime.state.StateBackend; +import org.apache.flink.runtime.state.AbstractStateBackend; import org.apache.flink.streaming.api.watermark.Watermark; import org.apache.flink.streaming.api.windowing.windows.TimeWindow; import org.apache.flink.streaming.runtime.operators.Triggerable; @@ -252,7 +252,7 @@ public StreamTaskState snapshotOperatorState(long checkpointId, long timestamp) // we write the panes with the key/value maps into the stream, as well as when this state // should have triggered and slided - StateBackend.CheckpointStateOutputView out = + AbstractStateBackend.CheckpointStateOutputView out = getStateBackend().createCheckpointStateOutputView(checkpointId, timestamp); out.writeLong(nextEvaluationTime); diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/NonKeyedWindowOperator.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/NonKeyedWindowOperator.java index 782363139e8b3..cce56573e5966 100644 --- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/NonKeyedWindowOperator.java +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/NonKeyedWindowOperator.java @@ -19,12 +19,12 @@ import com.google.common.annotations.VisibleForTesting; import org.apache.flink.api.common.ExecutionConfig; -import org.apache.flink.api.common.state.OperatorState; +import org.apache.flink.api.common.state.ValueState; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.common.typeutils.TypeSerializer; import org.apache.flink.api.java.typeutils.InputTypeConfigurable; import org.apache.flink.core.memory.DataInputView; -import org.apache.flink.runtime.state.StateBackend; +import org.apache.flink.runtime.state.AbstractStateBackend; import org.apache.flink.runtime.state.StateHandle; import org.apache.flink.streaming.api.functions.windowing.AllWindowFunction; import org.apache.flink.streaming.api.operators.AbstractUdfStreamOperator; @@ -397,7 +397,7 @@ protected Context(DataInputView in, ClassLoader userClassloader) throws Exceptio } } - protected void writeToState(StateBackend.CheckpointStateOutputView out) throws IOException { + protected void writeToState(AbstractStateBackend.CheckpointStateOutputView out) throws IOException { windowSerializer.serialize(window, out); out.writeLong(watermarkTimer); out.writeLong(processingTimeTimer); @@ -414,8 +414,8 @@ protected void writeToState(StateBackend.CheckpointStateOutputView out) throws I } @SuppressWarnings("unchecked") - public OperatorState getKeyValueState(final String name, final S defaultState) { - return new OperatorState() { + public ValueState getKeyValueState(final String name, final S defaultState) { + return new ValueState() { @Override public S value() throws IOException { Serializable value = state.get(name); @@ -430,6 +430,11 @@ public S value() throws IOException { public void update(S value) throws IOException { state.put(name, value); } + + @Override + public void clear() { + state.remove(name); + } }; } @@ -523,7 +528,7 @@ public StreamTaskState snapshotOperatorState(long checkpointId, long timestamp) StreamTaskState taskState = super.snapshotOperatorState(checkpointId, timestamp); // we write the panes with the key/value maps into the stream - StateBackend.CheckpointStateOutputView out = getStateBackend().createCheckpointStateOutputView(checkpointId, timestamp); + AbstractStateBackend.CheckpointStateOutputView out = getStateBackend().createCheckpointStateOutputView(checkpointId, timestamp); int numWindows = windows.size(); out.writeInt(numWindows); diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/WindowOperator.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/WindowOperator.java index 68c3a5f26bf13..46170b5176a97 100644 --- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/WindowOperator.java +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/WindowOperator.java @@ -19,13 +19,13 @@ import com.google.common.annotations.VisibleForTesting; import org.apache.flink.api.common.ExecutionConfig; -import org.apache.flink.api.common.state.OperatorState; +import org.apache.flink.api.common.state.ValueState; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.common.typeutils.TypeSerializer; import org.apache.flink.api.java.functions.KeySelector; import org.apache.flink.api.java.typeutils.InputTypeConfigurable; import org.apache.flink.core.memory.DataInputView; -import org.apache.flink.runtime.state.StateBackend; +import org.apache.flink.runtime.state.AbstractStateBackend; import org.apache.flink.runtime.state.StateHandle; import org.apache.flink.streaming.api.functions.windowing.WindowFunction; import org.apache.flink.streaming.api.operators.AbstractUdfStreamOperator; @@ -300,7 +300,7 @@ protected void emitWindow(Context context) throws Exception { if (context.windowBuffer.size() > 0) { - setKeyContextElement(context.windowBuffer.getElements().iterator().next()); + setKeyContextElement1(context.windowBuffer.getElements().iterator().next()); userFunction.apply(context.key, context.window, @@ -439,7 +439,7 @@ public Context(K key, /** * Constructs a new {@code Context} by reading from a {@link DataInputView} that * contains a serialized context that we wrote in - * {@link #writeToState(StateBackend.CheckpointStateOutputView)} + * {@link #writeToState(AbstractStateBackend.CheckpointStateOutputView)} */ @SuppressWarnings("unchecked") protected Context(DataInputView in, ClassLoader userClassloader) throws Exception { @@ -464,7 +464,7 @@ protected Context(DataInputView in, ClassLoader userClassloader) throws Exceptio /** * Writes the {@code Context} to the given state checkpoint output. */ - protected void writeToState(StateBackend.CheckpointStateOutputView out) throws IOException { + protected void writeToState(AbstractStateBackend.CheckpointStateOutputView out) throws IOException { keySerializer.serialize(key, out); windowSerializer.serialize(window, out); out.writeLong(watermarkTimer); @@ -482,8 +482,8 @@ protected void writeToState(StateBackend.CheckpointStateOutputView out) throws I } @SuppressWarnings("unchecked") - public OperatorState getKeyValueState(final String name, final S defaultState) { - return new OperatorState() { + public ValueState getKeyValueState(final String name, final S defaultState) { + return new ValueState() { @Override public S value() throws IOException { Serializable value = state.get(name); @@ -498,6 +498,11 @@ public S value() throws IOException { public void update(S value) throws IOException { state.put(name, value); } + + @Override + public void clear() { + state.remove(name); + } }; } @@ -591,7 +596,7 @@ public StreamTaskState snapshotOperatorState(long checkpointId, long timestamp) StreamTaskState taskState = super.snapshotOperatorState(checkpointId, timestamp); // we write the panes with the key/value maps into the stream - StateBackend.CheckpointStateOutputView out = getStateBackend().createCheckpointStateOutputView(checkpointId, timestamp); + AbstractStateBackend.CheckpointStateOutputView out = getStateBackend().createCheckpointStateOutputView(checkpointId, timestamp); int numKeys = windows.size(); out.writeInt(numKeys); diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/OperatorChain.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/OperatorChain.java index ac27093f5d949..125279c830e11 100644 --- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/OperatorChain.java +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/OperatorChain.java @@ -268,7 +268,7 @@ public ChainingOutput(OneInputStreamOperator operator) { @Override public void collect(StreamRecord record) { try { - operator.setKeyContextElement(record); + operator.setKeyContextElement1(record); operator.processElement(record); } catch (Exception e) { @@ -312,7 +312,7 @@ public void collect(StreamRecord record) { StreamRecord copy = new StreamRecord<>(serializer.copy(record.getValue()), record.getTimestamp()); - operator.setKeyContextElement(copy); + operator.setKeyContextElement1(copy); operator.processElement(copy); } catch (Exception e) { diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/StreamTask.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/StreamTask.java index 91f11fa0efac4..609e38d556c35 100644 --- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/StreamTask.java +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/StreamTask.java @@ -25,6 +25,7 @@ import java.util.concurrent.TimeUnit; import org.apache.flink.api.common.accumulators.Accumulator; +import org.apache.flink.api.common.typeutils.TypeSerializer; import org.apache.flink.configuration.ConfigConstants; import org.apache.flink.configuration.Configuration; import org.apache.flink.configuration.IllegalConfigurationException; @@ -37,11 +38,10 @@ import org.apache.flink.runtime.state.StateHandle; import org.apache.flink.runtime.taskmanager.DispatcherThreadFactory; import org.apache.flink.runtime.util.event.EventListener; -import org.apache.flink.streaming.api.checkpoint.CheckpointNotifier; import org.apache.flink.streaming.api.graph.StreamConfig; import org.apache.flink.streaming.api.operators.Output; import org.apache.flink.streaming.api.operators.StreamOperator; -import org.apache.flink.runtime.state.StateBackend; +import org.apache.flink.runtime.state.AbstractStateBackend; import org.apache.flink.runtime.state.StateBackendFactory; import org.apache.flink.runtime.state.filesystem.FsStateBackend; import org.apache.flink.runtime.state.filesystem.FsStateBackendFactory; @@ -122,9 +122,6 @@ public abstract class StreamTask> /** The class loader used to load dynamic classes of a job */ private ClassLoader userClassLoader; - /** The state backend that stores the state and checkpoints for this task */ - private StateBackend stateBackend; - /** The executor service that schedules and calls the triggers of this task*/ private ScheduledExecutorService timerService; @@ -215,11 +212,7 @@ public final void invoke() throws Exception { boolean disposed = false; try { - // first order of business is to initialize the state backend and to - // give operators back their state - stateBackend = createStateBackend(); - stateBackend.initializeForJob(getEnvironment()); - + // first order of business is to give operators back their state restoreStateLazy(); // we need to make sure that any triggers scheduled in open() cannot be @@ -283,14 +276,6 @@ public final void invoke() throws Exception { if (!disposed) { disposeAllOperators(); } - - try { - if (stateBackend != null) { - stateBackend.close(); - } - } catch (Throwable t) { - LOG.error("Error while closing the state backend", t); - } } } @@ -542,11 +527,6 @@ public void notifyCheckpointComplete(long checkpointId) throws Exception { if (isRunning) { LOG.debug("Notification of complete checkpoint for task {}", getName()); - // We first notify the state backend if necessary - if (stateBackend instanceof CheckpointNotifier) { - ((CheckpointNotifier) stateBackend).notifyCheckpointComplete(checkpointId); - } - for (StreamOperator operator : operatorChain.getAllOperators()) { if (operator != null) { operator.notifyOfCompletedCheckpoint(checkpointId); @@ -563,23 +543,12 @@ public void notifyCheckpointComplete(long checkpointId) throws Exception { // State backend // ------------------------------------------------------------------------ - /** - * Gets the state backend used by this task. The state backend defines how to maintain the - * key/value state and how and where to store state snapshots. - * - * @return The state backend used by this task. - */ - public StateBackend getStateBackend() { - return stateBackend; - } - - private StateBackend createStateBackend() throws Exception { - StateBackend configuredBackend = configuration.getStateBackend(userClassLoader); + public AbstractStateBackend createStateBackend(String operatorIdentifier, TypeSerializer keySerializer) throws Exception { + AbstractStateBackend stateBackend = configuration.getStateBackend(userClassLoader); - if (configuredBackend != null) { + if (stateBackend != null) { // backend has been configured on the environment - LOG.info("Using user-defined state backend: " + configuredBackend); - return configuredBackend; + LOG.info("Using user-defined state backend: " + stateBackend); } else { // see if we have a backend specified in the configuration Configuration flinkConfig = getEnvironment().getTaskManagerInfo().getConfiguration(); @@ -594,13 +563,15 @@ private StateBackend createStateBackend() throws Exception { switch (backendName) { case "jobmanager": LOG.info("State backend is set to heap memory (checkpoint to jobmanager)"); - return MemoryStateBackend.defaultInstance(); + stateBackend = MemoryStateBackend.create(); + break; case "filesystem": FsStateBackend backend = new FsStateBackendFactory().createFromConfig(flinkConfig); - LOG.info("State backend is set to filesystem (checkpoints to filesystem \"" + LOG.info("State backend is set to heap memory (checkpoints to filesystem \"" + backend.getBasePath() + "\")"); - return backend; + stateBackend = backend; + break; default: try { @@ -608,7 +579,7 @@ private StateBackend createStateBackend() throws Exception { Class clazz = Class.forName(backendName, false, userClassLoader).asSubclass(StateBackendFactory.class); - return clazz.newInstance().createFromConfig(flinkConfig); + stateBackend = ((StateBackendFactory) clazz.newInstance()).createFromConfig(flinkConfig); } catch (ClassNotFoundException e) { throw new IllegalConfigurationException("Cannot find configured state backend: " + backendName); } catch (ClassCastException e) { @@ -620,6 +591,9 @@ private StateBackend createStateBackend() throws Exception { } } } + stateBackend.initializeForJob(getEnvironment(), operatorIdentifier, keySerializer); + return stateBackend; + } /** diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/StreamTaskState.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/StreamTaskState.java index afeabd9ed6f5d..ace9cfd2a4d62 100644 --- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/StreamTaskState.java +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/StreamTaskState.java @@ -43,7 +43,7 @@ public class StreamTaskState implements Serializable { private StateHandle functionState; - private HashMap> kvStates; + private HashMap> kvStates; // ------------------------------------------------------------------------ @@ -63,11 +63,11 @@ public void setFunctionState(StateHandle functionState) { this.functionState = functionState; } - public HashMap> getKvStates() { + public HashMap> getKvStates() { return kvStates; } - public void setKvStates(HashMap> kvStates) { + public void setKvStates(HashMap> kvStates) { this.kvStates = kvStates; } @@ -92,7 +92,7 @@ public boolean isEmpty() { public void discardState() throws Exception { StateHandle operatorState = this.operatorState; StateHandle functionState = this.functionState; - HashMap> kvStates = this.kvStates; + HashMap> kvStates = this.kvStates; if (operatorState != null) { operatorState.discardState(); @@ -103,9 +103,9 @@ public void discardState() throws Exception { if (kvStates != null) { while (kvStates.size() > 0) { try { - Iterator> values = kvStates.values().iterator(); + Iterator> values = kvStates.values().iterator(); while (values.hasNext()) { - KvStateSnapshot s = values.next(); + KvStateSnapshot s = values.next(); s.discardState(); values.remove(); } @@ -121,4 +121,3 @@ public void discardState() throws Exception { this.kvStates = null; } } - \ No newline at end of file diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/StreamTaskStateList.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/StreamTaskStateList.java index e9f9ab6f9e5b1..e698db6d6e4ed 100644 --- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/StreamTaskStateList.java +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/StreamTaskStateList.java @@ -45,7 +45,7 @@ public StreamTaskStateList(StreamTaskState[] states) throws Exception { if (state != null) { StateHandle operatorState = state.getOperatorState(); StateHandle functionState = state.getFunctionState(); - HashMap> kvStates = state.getKvStates(); + HashMap> kvStates = state.getKvStates(); if (operatorState != null) { sumStateSize += operatorState.getStateSize(); @@ -56,7 +56,7 @@ public StreamTaskStateList(StreamTaskState[] states) throws Exception { } if (kvStates != null) { - for (KvStateSnapshot kvState : kvStates.values()) { + for (KvStateSnapshot kvState : kvStates.values()) { if (kvState != null) { sumStateSize += kvState.getStateSize(); } diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/DataStreamTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/DataStreamTest.java index 169c93d3decd1..475a95df67271 100644 --- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/DataStreamTest.java +++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/DataStreamTest.java @@ -653,7 +653,7 @@ public void sinkKeyTest() { StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); DataStreamSink sink = env.generateSequence(1, 100).print(); - assertTrue(env.getStreamGraph().getStreamNode(sink.getTransformation().getId()).getStatePartitioner() == null); + assertTrue(env.getStreamGraph().getStreamNode(sink.getTransformation().getId()).getStatePartitioner1() == null); assertTrue(env.getStreamGraph().getStreamNode(sink.getTransformation().getId()).getInEdges().get(0).getPartitioner() instanceof ForwardPartitioner); KeySelector key1 = new KeySelector() { @@ -668,10 +668,10 @@ public Long getKey(Long value) throws Exception { DataStreamSink sink2 = env.generateSequence(1, 100).keyBy(key1).print(); - assertNotNull(env.getStreamGraph().getStreamNode(sink2.getTransformation().getId()).getStatePartitioner()); + assertNotNull(env.getStreamGraph().getStreamNode(sink2.getTransformation().getId()).getStatePartitioner1()); assertNotNull(env.getStreamGraph().getStreamNode(sink2.getTransformation().getId()).getStateKeySerializer()); assertNotNull(env.getStreamGraph().getStreamNode(sink2.getTransformation().getId()).getStateKeySerializer()); - assertEquals(key1, env.getStreamGraph().getStreamNode(sink2.getTransformation().getId()).getStatePartitioner()); + assertEquals(key1, env.getStreamGraph().getStreamNode(sink2.getTransformation().getId()).getStatePartitioner1()); assertTrue(env.getStreamGraph().getStreamNode(sink2.getTransformation().getId()).getInEdges().get(0).getPartitioner() instanceof HashPartitioner); KeySelector key2 = new KeySelector() { @@ -686,8 +686,8 @@ public Long getKey(Long value) throws Exception { DataStreamSink sink3 = env.generateSequence(1, 100).keyBy(key2).print(); - assertTrue(env.getStreamGraph().getStreamNode(sink3.getTransformation().getId()).getStatePartitioner() != null); - assertEquals(key2, env.getStreamGraph().getStreamNode(sink3.getTransformation().getId()).getStatePartitioner()); + assertTrue(env.getStreamGraph().getStreamNode(sink3.getTransformation().getId()).getStatePartitioner1() != null); + assertEquals(key2, env.getStreamGraph().getStreamNode(sink3.getTransformation().getId()).getStatePartitioner1()); assertTrue(env.getStreamGraph().getStreamNode(sink3.getTransformation().getId()).getInEdges().get(0).getPartitioner() instanceof HashPartitioner); } diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/co/SelfConnectionTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/co/SelfConnectionTest.java index d00dc675cd539..8f04d41e1b574 100644 --- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/co/SelfConnectionTest.java +++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/co/SelfConnectionTest.java @@ -134,13 +134,13 @@ public Integer getKey(String value) throws Exception { public Long map(Integer value) throws Exception { return Long.valueOf(value + 1); } - }).keyBy(new KeySelector() { + }).keyBy(new KeySelector() { private static final long serialVersionUID = 1L; @Override - public Long getKey(Long value) throws Exception { - return value; + public Integer getKey(Long value) throws Exception { + return value.intValue(); } }); diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/AccumulatingAlignedProcessingTimeWindowOperatorTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/AccumulatingAlignedProcessingTimeWindowOperatorTest.java index ed0f04a8b9989..0e7001c314d49 100644 --- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/AccumulatingAlignedProcessingTimeWindowOperatorTest.java +++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/AccumulatingAlignedProcessingTimeWindowOperatorTest.java @@ -33,7 +33,7 @@ import org.apache.flink.streaming.api.functions.windowing.WindowFunction; import org.apache.flink.streaming.api.graph.StreamConfig; import org.apache.flink.streaming.api.operators.Output; -import org.apache.flink.runtime.state.StateBackend; +import org.apache.flink.runtime.state.AbstractStateBackend; import org.apache.flink.runtime.state.memory.MemoryStateBackend; import org.apache.flink.streaming.api.windowing.windows.TimeWindow; import org.apache.flink.streaming.runtime.operators.Triggerable; @@ -46,7 +46,6 @@ import org.junit.Test; import org.mockito.invocation.InvocationOnMock; import org.mockito.stubbing.Answer; -import org.mockito.stubbing.OngoingStubbing; import java.util.ArrayList; import java.util.Arrays; @@ -885,18 +884,27 @@ public void apply(Integer key, when(task.getName()).thenReturn("Test task name"); when(task.getExecutionConfig()).thenReturn(new ExecutionConfig()); - Environment env = mock(Environment.class); + final Environment env = mock(Environment.class); when(env.getTaskInfo()).thenReturn(new TaskInfo("Test task name", 0, 1, 0)); when(env.getUserClassLoader()).thenReturn(AggregatingAlignedProcessingTimeWindowOperatorTest.class.getClassLoader()); when(task.getEnvironment()).thenReturn(env); - // ugly java generic hacks to get the state backend into the mock - @SuppressWarnings("unchecked") - OngoingStubbing> stubbing = - (OngoingStubbing>) (OngoingStubbing) when(task.getStateBackend()); - stubbing.thenReturn(MemoryStateBackend.defaultInstance()); - + try { + doAnswer(new Answer() { + @Override + public AbstractStateBackend answer(InvocationOnMock invocationOnMock) throws Throwable { + final String operatorIdentifier = (String) invocationOnMock.getArguments()[0]; + final TypeSerializer keySerializer = (TypeSerializer) invocationOnMock.getArguments()[1]; + MemoryStateBackend backend = MemoryStateBackend.create(); + backend.initializeForJob(env, operatorIdentifier, keySerializer); + return backend; + } + }).when(task).createStateBackend(any(String.class), any(TypeSerializer.class)); + } catch (Exception e) { + e.printStackTrace(); + } + return task; } @@ -931,7 +939,7 @@ public Object call() throws Exception { private static StreamConfig createTaskConfig(KeySelector partitioner, TypeSerializer keySerializer) { StreamConfig cfg = new StreamConfig(new Configuration()); - cfg.setStatePartitioner(partitioner); + cfg.setStatePartitioner(0, partitioner); cfg.setStateKeySerializer(keySerializer); return cfg; } diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/AggregatingAlignedProcessingTimeWindowOperatorTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/AggregatingAlignedProcessingTimeWindowOperatorTest.java index b3e59e51c9ba6..58be09736998d 100644 --- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/AggregatingAlignedProcessingTimeWindowOperatorTest.java +++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/AggregatingAlignedProcessingTimeWindowOperatorTest.java @@ -36,7 +36,7 @@ import org.apache.flink.runtime.execution.Environment; import org.apache.flink.streaming.api.graph.StreamConfig; import org.apache.flink.streaming.api.operators.Output; -import org.apache.flink.runtime.state.StateBackend; +import org.apache.flink.runtime.state.AbstractStateBackend; import org.apache.flink.runtime.state.memory.MemoryStateBackend; import org.apache.flink.streaming.runtime.operators.Triggerable; import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; @@ -47,7 +47,6 @@ import org.junit.Test; import org.mockito.invocation.InvocationOnMock; import org.mockito.stubbing.Answer; -import org.mockito.stubbing.OngoingStubbing; import java.util.ArrayList; import java.util.Arrays; @@ -263,7 +262,7 @@ public void testTumblingWindowUniqueElements() { for (int i = 0; i < numElements; i++) { synchronized (lock) { StreamRecord> next = new StreamRecord<>(new Tuple2<>(i, i)); - op.setKeyContextElement(next); + op.setKeyContextElement1(next); op.processElement(next); } Thread.sleep(1); @@ -324,7 +323,7 @@ public void testTumblingWindowDuplicateElements() { int val = ((int) nextTime) ^ ((int) (nextTime >>> 32)); StreamRecord> next = new StreamRecord<>(new Tuple2<>(val, val)); - op.setKeyContextElement(next); + op.setKeyContextElement1(next); op.processElement(next); if (nextTime != previousNextTime) { @@ -383,7 +382,7 @@ public void testSlidingWindow() { for (int i = 0; i < numElements; i++) { synchronized (lock) { StreamRecord> next = new StreamRecord<>(new Tuple2<>(i, i)); - op.setKeyContextElement(next); + op.setKeyContextElement1(next); op.processElement(next); } Thread.sleep(1); @@ -449,11 +448,11 @@ public void testSlidingWindowSingleElements() { synchronized (lock) { StreamRecord> next1 = new StreamRecord<>(new Tuple2<>(1, 1)); - op.setKeyContextElement(next1); + op.setKeyContextElement1(next1); op.processElement(next1); StreamRecord> next2 = new StreamRecord<>(new Tuple2<>(2, 2)); - op.setKeyContextElement(next2); + op.setKeyContextElement1(next2); op.processElement(next2); } @@ -510,7 +509,7 @@ public void testEmitTrailingDataOnClose() { for (Integer i : data) { synchronized (lock) { StreamRecord> next = new StreamRecord<>(new Tuple2<>(i, i)); - op.setKeyContextElement(next); + op.setKeyContextElement1(next); op.processElement(next); } } @@ -563,14 +562,14 @@ public void testPropagateExceptionsFromProcessElement() { for (int i = 0; i < 100; i++) { synchronized (lock) { StreamRecord> next = new StreamRecord<>(new Tuple2<>(1, 1)); - op.setKeyContextElement(next); + op.setKeyContextElement1(next); op.processElement(next); } } try { StreamRecord> next = new StreamRecord<>(new Tuple2<>(1, 1)); - op.setKeyContextElement(next); + op.setKeyContextElement1(next); op.processElement(next); fail("This fail with an exception"); } @@ -615,7 +614,7 @@ public void checkpointRestoreWithPendingWindowTumbling() { for (int i = 0; i < numElementsFirst; i++) { synchronized (lock) { StreamRecord> next = new StreamRecord<>(new Tuple2<>(i, i)); - op.setKeyContextElement(next); + op.setKeyContextElement1(next); op.processElement(next); } Thread.sleep(1); @@ -638,7 +637,7 @@ public void checkpointRestoreWithPendingWindowTumbling() { for (int i = numElementsFirst; i < numElements; i++) { synchronized (lock) { StreamRecord> next = new StreamRecord<>(new Tuple2<>(i, i)); - op.setKeyContextElement(next); + op.setKeyContextElement1(next); op.processElement(next); } Thread.sleep(1); @@ -661,7 +660,7 @@ public void checkpointRestoreWithPendingWindowTumbling() { for (int i = numElementsFirst; i < numElements; i++) { synchronized (lock) { StreamRecord> next = new StreamRecord<>(new Tuple2<>(i, i)); - op.setKeyContextElement(next); + op.setKeyContextElement1(next); op.processElement(next); } Thread.sleep(1); @@ -721,7 +720,7 @@ public void checkpointRestoreWithPendingWindowSliding() { for (int i = 0; i < numElementsFirst; i++) { synchronized (lock) { StreamRecord> next = new StreamRecord<>(new Tuple2<>(i, i)); - op.setKeyContextElement(next); + op.setKeyContextElement1(next); op.processElement(next); } Thread.sleep(1); @@ -744,7 +743,7 @@ public void checkpointRestoreWithPendingWindowSliding() { for (int i = numElementsFirst; i < numElements; i++) { synchronized (lock) { StreamRecord> next = new StreamRecord<>(new Tuple2<>(i, i)); - op.setKeyContextElement(next); + op.setKeyContextElement1(next); op.processElement(next); } Thread.sleep(1); @@ -768,7 +767,7 @@ public void checkpointRestoreWithPendingWindowSliding() { for (int i = numElementsFirst; i < numElements; i++) { synchronized (lock) { StreamRecord> next = new StreamRecord<>(new Tuple2<>(i, i)); - op.setKeyContextElement(next); + op.setKeyContextElement1(next); op.processElement(next); } Thread.sleep(1); @@ -834,11 +833,11 @@ public void testKeyValueStateInWindowFunctionTumbling() { synchronized (lock) { for (int i = 0; i < 10; i++) { StreamRecord> next1 = new StreamRecord<>(new Tuple2<>(1, i)); - op.setKeyContextElement(next1); + op.setKeyContextElement1(next1); op.processElement(next1); StreamRecord> next2 = new StreamRecord<>(new Tuple2<>(2, i)); - op.setKeyContextElement(next2); + op.setKeyContextElement1(next2); op.processElement(next2); } @@ -902,13 +901,13 @@ public void testKeyValueStateInWindowFunctionSliding() { // because we do not release the lock between elements, they end up in the same windows synchronized (lock) { - op.setKeyContextElement(next1); + op.setKeyContextElement1(next1); op.processElement(next1); - op.setKeyContextElement(next2); + op.setKeyContextElement1(next2); op.processElement(next2); - op.setKeyContextElement(next3); + op.setKeyContextElement1(next3); op.processElement(next3); - op.setKeyContextElement(next4); + op.setKeyContextElement1(next4); op.processElement(next4); } @@ -1012,18 +1011,27 @@ public Tuple2 reduce(Tuple2 value1, Tuple2> stubbing = - (OngoingStubbing>) (OngoingStubbing) when(task.getStateBackend()); - stubbing.thenReturn(MemoryStateBackend.defaultInstance()); - + try { + doAnswer(new Answer() { + @Override + public AbstractStateBackend answer(InvocationOnMock invocationOnMock) throws Throwable { + final String operatorIdentifier = (String) invocationOnMock.getArguments()[0]; + final TypeSerializer keySerializer = (TypeSerializer) invocationOnMock.getArguments()[1]; + MemoryStateBackend backend = MemoryStateBackend.create(); + backend.initializeForJob(env, operatorIdentifier, keySerializer); + return backend; + } + }).when(task).createStateBackend(any(String.class), any(TypeSerializer.class)); + } catch (Exception e) { + e.printStackTrace(); + } + return task; } @@ -1058,7 +1066,7 @@ public Object call() throws Exception { private static StreamConfig createTaskConfig(KeySelector partitioner, TypeSerializer keySerializer) { StreamConfig cfg = new StreamConfig(new Configuration()); - cfg.setStatePartitioner(partitioner); + cfg.setStatePartitioner(0, partitioner); cfg.setStateKeySerializer(keySerializer); return cfg; } diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/util/MockContext.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/util/MockContext.java index 0c708c656cae5..675e7b6b91a3e 100644 --- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/util/MockContext.java +++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/util/MockContext.java @@ -29,21 +29,22 @@ import org.apache.flink.api.common.ExecutionConfig; import org.apache.flink.api.common.accumulators.Accumulator; import org.apache.flink.api.common.typeinfo.TypeInformation; +import org.apache.flink.api.common.typeutils.TypeSerializer; import org.apache.flink.api.java.functions.KeySelector; import org.apache.flink.configuration.Configuration; +import org.apache.flink.runtime.operators.testutils.DummyEnvironment; import org.apache.flink.runtime.operators.testutils.MockEnvironment; import org.apache.flink.runtime.operators.testutils.MockInputSplitProvider; import org.apache.flink.streaming.api.graph.StreamConfig; import org.apache.flink.streaming.api.operators.OneInputStreamOperator; import org.apache.flink.streaming.api.operators.Output; -import org.apache.flink.runtime.state.StateBackend; +import org.apache.flink.runtime.state.AbstractStateBackend; import org.apache.flink.runtime.state.memory.MemoryStateBackend; import org.apache.flink.streaming.runtime.operators.Triggerable; import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; import org.apache.flink.streaming.runtime.tasks.StreamTask; import org.mockito.invocation.InvocationOnMock; import org.mockito.stubbing.Answer; -import org.mockito.stubbing.OngoingStubbing; import static org.mockito.Matchers.any; import static org.mockito.Matchers.anyLong; @@ -87,13 +88,13 @@ public static List createAndExecuteForKeyedStream( StreamConfig config = new StreamConfig(new Configuration()); if (keySelector != null && keyType != null) { config.setStateKeySerializer(keyType.createSerializer(new ExecutionConfig())); - config.setStatePartitioner(keySelector); + config.setStatePartitioner(0, keySelector); } final ScheduledExecutorService timerService = Executors.newSingleThreadScheduledExecutor(); final Object lock = new Object(); final StreamTask mockTask = createMockTaskWithTimer(timerService, lock); - + operator.setup(mockTask, config, mockContext.output); try { operator.open(); @@ -102,7 +103,7 @@ public static List createAndExecuteForKeyedStream( for (IN in: inputs) { record = record.replace(in); synchronized (lock) { - operator.setKeyContextElement(record); + operator.setKeyContextElement1(record); operator.processElement(record); } } @@ -148,12 +149,22 @@ public Object call() throws Exception { } }).when(task).registerTimer(anyLong(), any(Triggerable.class)); - // ugly Java generic hacks to get the generic state backend into the mock - @SuppressWarnings("unchecked") - OngoingStubbing> stubbing = - (OngoingStubbing>) (OngoingStubbing) when(task.getStateBackend()); - stubbing.thenReturn(MemoryStateBackend.defaultInstance()); - + + try { + doAnswer(new Answer() { + @Override + public AbstractStateBackend answer(InvocationOnMock invocationOnMock) throws Throwable { + final String operatorIdentifier = (String) invocationOnMock.getArguments()[0]; + final TypeSerializer keySerializer = (TypeSerializer) invocationOnMock.getArguments()[1]; + MemoryStateBackend backend = MemoryStateBackend.create(); + backend.initializeForJob(new DummyEnvironment("dummty", 1, 0), operatorIdentifier, keySerializer); + return backend; + } + }).when(task).createStateBackend(any(String.class), any(TypeSerializer.class)); + } catch (Exception e) { + e.printStackTrace(); + } + return task; } } diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/util/OneInputStreamOperatorTestHarness.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/util/OneInputStreamOperatorTestHarness.java index 01f95bc5ccc0c..618bd2a9959da 100644 --- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/util/OneInputStreamOperatorTestHarness.java +++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/util/OneInputStreamOperatorTestHarness.java @@ -30,16 +30,19 @@ import org.apache.flink.streaming.api.graph.StreamConfig; import org.apache.flink.streaming.api.operators.OneInputStreamOperator; import org.apache.flink.streaming.api.operators.Output; -import org.apache.flink.runtime.state.StateBackend; +import org.apache.flink.runtime.state.AbstractStateBackend; import org.apache.flink.runtime.state.memory.MemoryStateBackend; import org.apache.flink.streaming.api.watermark.Watermark; import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; import org.apache.flink.streaming.runtime.tasks.StreamTask; -import org.mockito.stubbing.OngoingStubbing; +import org.mockito.invocation.InvocationOnMock; +import org.mockito.stubbing.Answer; import java.util.Collection; import java.util.concurrent.ConcurrentLinkedQueue; +import static org.mockito.Matchers.any; +import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -62,6 +65,10 @@ public class OneInputStreamOperatorTestHarness { final ExecutionConfig executionConfig; final Object checkpointLock; + + StreamTask mockTask; + + AbstractStateBackend stateBackend; public OneInputStreamOperatorTestHarness(OneInputStreamOperator operator) { @@ -71,26 +78,33 @@ public OneInputStreamOperatorTestHarness(OneInputStreamOperator operato this.executionConfig = new ExecutionConfig(); this.checkpointLock = new Object(); - Environment env = new MockEnvironment("MockTwoInputTask", 3 * 1024 * 1024, new MockInputSplitProvider(), 1024); - StreamTask mockTask = mock(StreamTask.class); + final Environment env = new MockEnvironment("MockTwoInputTask", 3 * 1024 * 1024, new MockInputSplitProvider(), 1024); + mockTask = mock(StreamTask.class); when(mockTask.getName()).thenReturn("Mock Task"); when(mockTask.getCheckpointLock()).thenReturn(checkpointLock); when(mockTask.getConfiguration()).thenReturn(config); when(mockTask.getEnvironment()).thenReturn(env); when(mockTask.getExecutionConfig()).thenReturn(executionConfig); - - // ugly Java generic hacks - @SuppressWarnings("unchecked") - OngoingStubbing> stubbing = - (OngoingStubbing>) (OngoingStubbing) when(mockTask.getStateBackend()); - stubbing.thenReturn(MemoryStateBackend.defaultInstance()); - operator.setup(mockTask, config, new MockOutput()); + try { + doAnswer(new Answer() { + @Override + public AbstractStateBackend answer(InvocationOnMock invocationOnMock) throws Throwable { + final String operatorIdentifier = (String) invocationOnMock.getArguments()[0]; + final TypeSerializer keySerializer = (TypeSerializer) invocationOnMock.getArguments()[1]; + MemoryStateBackend backend = MemoryStateBackend.create(); + backend.initializeForJob(env, operatorIdentifier, keySerializer); + return backend; + } + }).when(mockTask).createStateBackend(any(String.class), any(TypeSerializer.class)); + } catch (Exception e) { + e.printStackTrace(); + } } public void configureForKeyedStream(KeySelector keySelector, TypeInformation keyType) { ClosureCleaner.clean(keySelector, false); - config.setStatePartitioner(keySelector); + config.setStatePartitioner(0, keySelector); config.setStateKeySerializer(keyType.createSerializer(executionConfig)); } @@ -107,6 +121,8 @@ public ConcurrentLinkedQueue getOutput() { * Calls {@link org.apache.flink.streaming.api.operators.StreamOperator#open()} */ public void open() throws Exception { + operator.setup(mockTask, config, new MockOutput()); + operator.open(); } @@ -118,13 +134,13 @@ public void close() throws Exception { } public void processElement(StreamRecord element) throws Exception { - operator.setKeyContextElement(element); + operator.setKeyContextElement1(element); operator.processElement(element); } public void processElements(Collection> elements) throws Exception { for (StreamRecord element: elements) { - operator.setKeyContextElement(element); + operator.setKeyContextElement1(element); operator.processElement(element); } } diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/util/TwoInputStreamOperatorTestHarness.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/util/TwoInputStreamOperatorTestHarness.java index c586db36e3642..e23673a938e21 100644 --- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/util/TwoInputStreamOperatorTestHarness.java +++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/util/TwoInputStreamOperatorTestHarness.java @@ -28,7 +28,7 @@ import org.apache.flink.streaming.api.graph.StreamConfig; import org.apache.flink.streaming.api.operators.Output; import org.apache.flink.streaming.api.operators.TwoInputStreamOperator; -import org.apache.flink.runtime.state.StateBackend; +import org.apache.flink.runtime.state.AbstractStateBackend; import org.apache.flink.runtime.state.memory.MemoryStateBackend; import org.apache.flink.streaming.api.watermark.Watermark; import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; @@ -76,12 +76,6 @@ public TwoInputStreamOperatorTestHarness(TwoInputStreamOperator o when(mockTask.getEnvironment()).thenReturn(env); when(mockTask.getExecutionConfig()).thenReturn(executionConfig); - // ugly Java generic hacks - @SuppressWarnings("unchecked") - OngoingStubbing> stubbing = - (OngoingStubbing>) (OngoingStubbing) when(mockTask.getStateBackend()); - stubbing.thenReturn(MemoryStateBackend.defaultInstance()); - operator.setup(mockTask, new StreamConfig(new Configuration()), new MockOutput()); } diff --git a/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/StreamExecutionEnvironment.scala b/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/StreamExecutionEnvironment.scala index 69147f683155b..29bf5da33fe20 100644 --- a/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/StreamExecutionEnvironment.scala +++ b/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/StreamExecutionEnvironment.scala @@ -23,7 +23,8 @@ import org.apache.flink.api.common.io.{FileInputFormat, InputFormat} import org.apache.flink.api.common.typeinfo.TypeInformation import org.apache.flink.api.java.typeutils.runtime.kryo.KryoSerializer import org.apache.flink.api.scala.ClosureCleaner -import org.apache.flink.runtime.state.StateBackend +import org.apache.flink.runtime.state.AbstractStateBackend +import org.apache.flink.streaming.api.{TimeCharacteristic, CheckpointingMode} import org.apache.flink.streaming.api.environment.{StreamExecutionEnvironment => JavaEnv} import org.apache.flink.streaming.api.functions.source.FileMonitoringFunction.WatchType import org.apache.flink.streaming.api.functions.source.SourceFunction @@ -211,7 +212,7 @@ class StreamExecutionEnvironment(javaEnv: JavaEnv) { * program can be executed highly available and strongly consistent (assuming that Flink * is run in high-availability mode). */ - def setStateBackend(backend: StateBackend[_]): StreamExecutionEnvironment = { + def setStateBackend(backend: AbstractStateBackend): StreamExecutionEnvironment = { javaEnv.setStateBackend(backend) this } @@ -219,7 +220,7 @@ class StreamExecutionEnvironment(javaEnv: JavaEnv) { /** * Returns the state backend that defines how to store and checkpoint state. */ - def getStateBackend: StateBackend[_] = javaEnv.getStateBackend() + def getStateBackend: AbstractStateBackend = javaEnv.getStateBackend() /** * Sets the number of times that failed tasks are re-executed. A value of zero diff --git a/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/function/StatefulFunction.scala b/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/function/StatefulFunction.scala index d66cfdb7393e5..dc49173e50814 100644 --- a/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/function/StatefulFunction.scala +++ b/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/function/StatefulFunction.scala @@ -19,9 +19,9 @@ package org.apache.flink.streaming.api.scala.function import org.apache.flink.api.common.functions.RichFunction +import org.apache.flink.api.common.state.OperatorState import org.apache.flink.api.common.typeinfo.TypeInformation import org.apache.flink.configuration.Configuration -import org.apache.flink.api.common.state.OperatorState /** * Trait implementing the functionality necessary to apply stateful functions in diff --git a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/EventTimeAllWindowCheckpointingITCase.java b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/EventTimeAllWindowCheckpointingITCase.java index 304dcb5062d50..18c1b3c06d9fd 100644 --- a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/EventTimeAllWindowCheckpointingITCase.java +++ b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/EventTimeAllWindowCheckpointingITCase.java @@ -23,8 +23,8 @@ import org.apache.flink.api.java.tuple.Tuple4; import org.apache.flink.configuration.ConfigConstants; import org.apache.flink.configuration.Configuration; +import org.apache.flink.runtime.state.CheckpointListener; import org.apache.flink.streaming.api.TimeCharacteristic; -import org.apache.flink.streaming.api.checkpoint.CheckpointNotifier; import org.apache.flink.streaming.api.checkpoint.Checkpointed; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.flink.streaming.api.functions.sink.RichSinkFunction; @@ -383,7 +383,7 @@ public void apply( // ------------------------------------------------------------------------ private static class FailingSource extends RichEventTimeSourceFunction> - implements Checkpointed, CheckpointNotifier + implements Checkpointed, CheckpointListener { private static volatile boolean failedBefore = false; diff --git a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/EventTimeWindowCheckpointingITCase.java b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/EventTimeWindowCheckpointingITCase.java index 81e8f0af5089f..7a1a879c72769 100644 --- a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/EventTimeWindowCheckpointingITCase.java +++ b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/EventTimeWindowCheckpointingITCase.java @@ -25,8 +25,8 @@ import org.apache.flink.api.java.tuple.Tuple4; import org.apache.flink.configuration.ConfigConstants; import org.apache.flink.configuration.Configuration; +import org.apache.flink.runtime.state.CheckpointListener; import org.apache.flink.streaming.api.TimeCharacteristic; -import org.apache.flink.streaming.api.checkpoint.CheckpointNotifier; import org.apache.flink.streaming.api.checkpoint.Checkpointed; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.flink.streaming.api.functions.sink.RichSinkFunction; @@ -451,7 +451,7 @@ public void apply( // ------------------------------------------------------------------------ private static class FailingSource extends RichEventTimeSourceFunction> - implements Checkpointed, CheckpointNotifier + implements Checkpointed, CheckpointListener { private static volatile boolean failedBefore = false; diff --git a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/PartitionedStateCheckpointingITCase.java b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/PartitionedStateCheckpointingITCase.java index 42b62303d8b42..387421e0ebdeb 100644 --- a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/PartitionedStateCheckpointingITCase.java +++ b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/PartitionedStateCheckpointingITCase.java @@ -29,6 +29,9 @@ import org.apache.flink.api.common.functions.RichMapFunction; import org.apache.flink.api.common.state.OperatorState; +import org.apache.flink.api.common.state.ValueState; +import org.apache.flink.api.common.state.ValueStateDescriptor; +import org.apache.flink.api.common.typeutils.base.LongSerializer; import org.apache.flink.api.java.functions.KeySelector; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.configuration.Configuration; @@ -190,14 +193,17 @@ private static class CounterSink extends RichSinkFunction> private static Map allCounts = new ConcurrentHashMap(); + private ValueStateDescriptor bCountsId = new ValueStateDescriptor<>("b", 0L, + LongSerializer.INSTANCE); + private OperatorState aCounts; - private OperatorState bCounts; + private ValueState bCounts; @Override public void open(Configuration parameters) throws IOException { aCounts = getRuntimeContext().getKeyValueState( "a", NonSerializableLong.class, NonSerializableLong.of(0L)); - bCounts = getRuntimeContext().getKeyValueState("b", Long.class, 0L); + bCounts = getRuntimeContext().getPartitionedState(bCountsId); } @Override @@ -224,6 +230,22 @@ private NonSerializableLong(long value) { public static NonSerializableLong of(long value) { return new NonSerializableLong(value); } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + NonSerializableLong that = (NonSerializableLong) o; + + return value.equals(that.value); + + } + + @Override + public int hashCode() { + return value.hashCode(); + } } public static class IdentityKeySelector implements KeySelector { diff --git a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/SavepointITCase.java b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/SavepointITCase.java index 4e5e1b5d8a8bb..46c0453c752fe 100644 --- a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/SavepointITCase.java +++ b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/SavepointITCase.java @@ -44,7 +44,7 @@ import org.apache.flink.runtime.messages.JobManagerMessages.DisposeSavepoint; import org.apache.flink.runtime.messages.JobManagerMessages.TriggerSavepoint; import org.apache.flink.runtime.messages.JobManagerMessages.TriggerSavepointSuccess; -import org.apache.flink.runtime.state.filesystem.AbstractFileState; +import org.apache.flink.runtime.state.filesystem.AbstractFileStateHandle; import org.apache.flink.runtime.state.filesystem.FsStateBackend; import org.apache.flink.runtime.state.filesystem.FsStateBackendFactory; import org.apache.flink.runtime.testingUtils.TestingJobManagerMessages.NotifyWhenJobRemoved; @@ -53,7 +53,7 @@ import org.apache.flink.runtime.testingUtils.TestingTaskManagerMessages; import org.apache.flink.runtime.testingUtils.TestingTaskManagerMessages.ResponseSubmitTaskListener; import org.apache.flink.runtime.testutils.CommonTestUtils; -import org.apache.flink.streaming.api.checkpoint.CheckpointNotifier; +import org.apache.flink.runtime.state.CheckpointListener; import org.apache.flink.streaming.api.checkpoint.Checkpointed; import org.apache.flink.streaming.api.datastream.DataStream; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; @@ -365,7 +365,7 @@ protected void run() { for (StreamTaskState taskState : taskStateList.getState( ClassLoader.getSystemClassLoader())) { - AbstractFileState fsState = (AbstractFileState) taskState.getFunctionState(); + AbstractFileStateHandle fsState = (AbstractFileStateHandle) taskState.getFunctionState(); checkpointFiles.add(new File(fsState.getFilePath().toUri())); } } @@ -660,7 +660,7 @@ public void testCheckpointsRemovedWithJobManagerBackendOnShutdown() throws Excep for (StreamTaskState taskState : taskStateList.getState( ClassLoader.getSystemClassLoader())) { - AbstractFileState fsState = (AbstractFileState) taskState.getFunctionState(); + AbstractFileStateHandle fsState = (AbstractFileStateHandle) taskState.getFunctionState(); checkpointFiles.add(new File(fsState.getFilePath().toUri())); } } @@ -784,7 +784,7 @@ public void invoke(Integer value) throws Exception { } private static class InfiniteTestSource - implements SourceFunction, CheckpointNotifier { + implements SourceFunction, CheckpointListener { private static final long serialVersionUID = 1L; private volatile boolean running = true; diff --git a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/StateCheckpointedITCase.java b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/StateCheckpointedITCase.java index d7c06f67ae4d8..962fe8403d413 100644 --- a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/StateCheckpointedITCase.java +++ b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/StateCheckpointedITCase.java @@ -21,9 +21,9 @@ import org.apache.flink.api.common.functions.RichFilterFunction; import org.apache.flink.api.common.functions.RichFlatMapFunction; import org.apache.flink.api.common.functions.RichMapFunction; -import org.apache.flink.api.common.state.OperatorState; +import org.apache.flink.api.common.state.ValueState; import org.apache.flink.configuration.Configuration; -import org.apache.flink.streaming.api.checkpoint.CheckpointNotifier; +import org.apache.flink.runtime.state.CheckpointListener; import org.apache.flink.streaming.api.checkpoint.Checkpointed; import org.apache.flink.streaming.api.checkpoint.CheckpointedAsynchronously; import org.apache.flink.streaming.api.datastream.DataStream; @@ -47,7 +47,7 @@ * A simple test that runs a streaming topology with checkpointing enabled. * * The test triggers a failure after a while and verifies that, after completion, the - * state defined with either the {@link OperatorState} or the {@link Checkpointed} + * state defined with either the {@link ValueState} or the {@link Checkpointed} * interface reflects the "exactly once" semantics. * * The test throttles the input until at least two checkpoints are completed, to make sure that @@ -295,7 +295,7 @@ public void restoreState(Long state) { } private static class OnceFailingAggregator extends RichFlatMapFunction - implements Checkpointed>, CheckpointNotifier { + implements Checkpointed>, CheckpointListener { static boolean wasCheckpointedBeforeFailure = false; diff --git a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/StreamCheckpointNotifierITCase.java b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/StreamCheckpointNotifierITCase.java index 22f61b72cbfac..5fa066686826b 100644 --- a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/StreamCheckpointNotifierITCase.java +++ b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/StreamCheckpointNotifierITCase.java @@ -24,7 +24,7 @@ import org.apache.flink.api.java.tuple.Tuple1; import org.apache.flink.configuration.ConfigConstants; import org.apache.flink.configuration.Configuration; -import org.apache.flink.streaming.api.checkpoint.CheckpointNotifier; +import org.apache.flink.runtime.state.CheckpointListener; import org.apache.flink.streaming.api.checkpoint.Checkpointed; import org.apache.flink.streaming.api.datastream.DataStream; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; @@ -55,8 +55,8 @@ import static org.junit.Assert.fail; /** - * Integration test for the {@link CheckpointNotifier} interface. The test ensures that - * {@link CheckpointNotifier#notifyCheckpointComplete(long)} is called for completed + * Integration test for the {@link CheckpointListener} interface. The test ensures that + * {@link CheckpointListener#notifyCheckpointComplete(long)} is called for completed * checkpoints, that it is called at most once for any checkpoint id and that it is not * called for a deliberately failed checkpoint. * @@ -66,7 +66,7 @@ * *

* Note that as a result of doing the checks on the task level there is no way to verify - * that the {@link CheckpointNotifier#notifyCheckpointComplete(long)} is called for every + * that the {@link CheckpointListener#notifyCheckpointComplete(long)} is called for every * successfully completed checkpoint. */ @SuppressWarnings("serial") @@ -197,11 +197,11 @@ static List[] createCheckpointLists(int parallelism) { // -------------------------------------------------------------------------------------------- /** - * Generates some Long values and as an implementation for the {@link CheckpointNotifier} + * Generates some Long values and as an implementation for the {@link CheckpointListener} * interface it stores all the checkpoint ids it has seen in a static list. */ private static class GeneratingSourceFunction extends RichSourceFunction - implements ParallelSourceFunction, CheckpointNotifier, Checkpointed { + implements ParallelSourceFunction, CheckpointListener, Checkpointed { static final List[] completedCheckpoints = createCheckpointLists(PARALLELISM); @@ -285,10 +285,10 @@ public void notifyCheckpointComplete(long checkpointId) { /** * Identity transform on Long values wrapping the output in a tuple. As an implementation - * for the {@link CheckpointNotifier} interface it stores all the checkpoint ids it has seen in a static list. + * for the {@link CheckpointListener} interface it stores all the checkpoint ids it has seen in a static list. */ private static class IdentityMapFunction extends RichMapFunction> - implements CheckpointNotifier { + implements CheckpointListener { static final List[] completedCheckpoints = createCheckpointLists(PARALLELISM); @@ -316,10 +316,10 @@ public void notifyCheckpointComplete(long checkpointId) { /** * Filter on Long values supposedly letting all values through. As an implementation - * for the {@link CheckpointNotifier} interface it stores all the checkpoint ids + * for the {@link CheckpointListener} interface it stores all the checkpoint ids * it has seen in a static list. */ - private static class LongRichFilterFunction extends RichFilterFunction implements CheckpointNotifier { + private static class LongRichFilterFunction extends RichFilterFunction implements CheckpointListener { static final List[] completedCheckpoints = createCheckpointLists(PARALLELISM); @@ -347,11 +347,11 @@ public void notifyCheckpointComplete(long checkpointId) { /** * CoFlatMap on Long values as identity transform on the left input, while ignoring the right. - * As an implementation for the {@link CheckpointNotifier} interface it stores all the checkpoint + * As an implementation for the {@link CheckpointListener} interface it stores all the checkpoint * ids it has seen in a static list. */ private static class LeftIdentityCoRichFlatMapFunction extends RichCoFlatMapFunction - implements CheckpointNotifier { + implements CheckpointListener { static final List[] completedCheckpoints = createCheckpointLists(PARALLELISM); @@ -386,7 +386,7 @@ public void notifyCheckpointComplete(long checkpointId) { * Reducer that causes one failure between seeing 40% to 70% of the records. */ private static class OnceFailingReducer extends RichReduceFunction> - implements Checkpointed, CheckpointNotifier + implements Checkpointed, CheckpointListener { static volatile boolean hasFailed = false; static volatile long failureCheckpointID; diff --git a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/WindowCheckpointingITCase.java b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/WindowCheckpointingITCase.java index 500d7d33c0c85..8d59975bb4686 100644 --- a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/WindowCheckpointingITCase.java +++ b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/WindowCheckpointingITCase.java @@ -26,8 +26,8 @@ import org.apache.flink.configuration.ConfigConstants; import org.apache.flink.configuration.Configuration; import org.apache.flink.runtime.client.JobExecutionException; +import org.apache.flink.runtime.state.CheckpointListener; import org.apache.flink.streaming.api.TimeCharacteristic; -import org.apache.flink.streaming.api.checkpoint.CheckpointNotifier; import org.apache.flink.streaming.api.checkpoint.Checkpointed; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.flink.streaming.api.functions.sink.RichSinkFunction; @@ -335,7 +335,7 @@ public Tuple2 reduce( // ------------------------------------------------------------------------ private static class FailingSource extends RichSourceFunction> - implements Checkpointed, CheckpointNotifier + implements Checkpointed, CheckpointListener { private static volatile boolean failedBefore = false; diff --git a/flink-tests/src/test/java/org/apache/flink/test/classloading/jar/CheckpointedStreamingProgram.java b/flink-tests/src/test/java/org/apache/flink/test/classloading/jar/CheckpointedStreamingProgram.java index 47253da76f1b9..05b20d6c6e678 100644 --- a/flink-tests/src/test/java/org/apache/flink/test/classloading/jar/CheckpointedStreamingProgram.java +++ b/flink-tests/src/test/java/org/apache/flink/test/classloading/jar/CheckpointedStreamingProgram.java @@ -19,7 +19,7 @@ package org.apache.flink.test.classloading.jar; import org.apache.flink.api.common.functions.MapFunction; -import org.apache.flink.streaming.api.checkpoint.CheckpointNotifier; +import org.apache.flink.runtime.state.CheckpointListener; import org.apache.flink.streaming.api.checkpoint.Checkpointed; import org.apache.flink.streaming.api.datastream.DataStream; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; @@ -93,7 +93,7 @@ public void restoreState(Integer state) { } } - public static class StatefulMapper implements MapFunction, Checkpointed, CheckpointNotifier { + public static class StatefulMapper implements MapFunction, Checkpointed, CheckpointListener { private String someState; private boolean atLeastOneSnapshotComplete = false; diff --git a/flink-tests/src/test/java/org/apache/flink/test/recovery/ChaosMonkeyITCase.java b/flink-tests/src/test/java/org/apache/flink/test/recovery/ChaosMonkeyITCase.java index acc85699251fe..6ae0d46e865ea 100644 --- a/flink-tests/src/test/java/org/apache/flink/test/recovery/ChaosMonkeyITCase.java +++ b/flink-tests/src/test/java/org/apache/flink/test/recovery/ChaosMonkeyITCase.java @@ -41,7 +41,7 @@ import org.apache.flink.runtime.testutils.ZooKeeperTestUtils; import org.apache.flink.runtime.util.ZooKeeperUtils; import org.apache.flink.runtime.zookeeper.ZooKeeperTestEnvironment; -import org.apache.flink.streaming.api.checkpoint.CheckpointNotifier; +import org.apache.flink.runtime.state.CheckpointListener; import org.apache.flink.streaming.api.checkpoint.Checkpointed; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.flink.streaming.api.functions.sink.RichSinkFunction; @@ -378,7 +378,7 @@ private JobGraph createJobGraph( } public static class CheckpointedSequenceSource extends RichParallelSourceFunction - implements Checkpointed, CheckpointNotifier { + implements Checkpointed, CheckpointListener { private static final long serialVersionUID = 0L; @@ -448,7 +448,7 @@ public void notifyCheckpointComplete(long checkpointId) throws Exception { } public static class CountingSink extends RichSinkFunction - implements Checkpointed, CheckpointNotifier { + implements Checkpointed, CheckpointListener { private static final Logger LOG = LoggerFactory.getLogger(CountingSink.class); diff --git a/flink-tests/src/test/java/org/apache/flink/test/recovery/JobManagerCheckpointRecoveryITCase.java b/flink-tests/src/test/java/org/apache/flink/test/recovery/JobManagerCheckpointRecoveryITCase.java index cc4998db46d0f..737d39ada2056 100644 --- a/flink-tests/src/test/java/org/apache/flink/test/recovery/JobManagerCheckpointRecoveryITCase.java +++ b/flink-tests/src/test/java/org/apache/flink/test/recovery/JobManagerCheckpointRecoveryITCase.java @@ -40,7 +40,7 @@ import org.apache.flink.runtime.testutils.ZooKeeperTestUtils; import org.apache.flink.runtime.util.ZooKeeperUtils; import org.apache.flink.runtime.zookeeper.ZooKeeperTestEnvironment; -import org.apache.flink.streaming.api.checkpoint.CheckpointNotifier; +import org.apache.flink.runtime.state.CheckpointListener; import org.apache.flink.streaming.api.checkpoint.Checkpointed; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.flink.streaming.api.functions.sink.SinkFunction; @@ -505,7 +505,7 @@ public void cancel() { * are exhausted. */ public static class CountingSink implements SinkFunction, Checkpointed, - CheckpointNotifier { + CheckpointListener { private static final Logger LOG = LoggerFactory.getLogger(CountingSink.class); From c949a194e0f4fcc72df268aed228b90217f327d9 Mon Sep 17 00:00:00 2001 From: Aljoscha Krettek Date: Mon, 25 Jan 2016 12:34:05 +0100 Subject: [PATCH 2/3] [FLINK-3200] Use Partitioned State in WindowOperator This changes window operator to use the new partitioned state abstraction for keeping window contents instead of custom internal state and the checkpointed interface. For now, timers are still kept as custom checkpointed state, however. WindowOperator now expects a StateIdentifier for MergingState, this can either be for ReducingState or ListState but WindowOperator is agnostic to the type of State. Also the signature of WindowFunction is changed to include the type of intermediate input. For example, if a ReducingState is used the input of the WindowFunction is T (where T is the input type). If using a ListState the input of the WindowFunction would be of type Iterable[T]. --- .../ml/IncrementalLearningSkeleton.java | 2 +- .../GroupedProcessingTimeWindowExample.java | 2 +- .../examples/windowing/SessionWindowing.java | 11 +- .../examples/windowing/TopSpeedWindowing.java | 2 +- .../util/TopSpeedWindowingExampleData.java | 8 +- .../windowing/TopSpeedWindowing.scala | 2 +- .../api/datastream/AllWindowedStream.java | 37 +- .../api/datastream/CoGroupedStreams.java | 2 +- .../api/datastream/WindowedStream.java | 118 ++-- .../aggregation/AggregationFunction.java | 4 +- .../windowing/AllWindowFunction.java | 4 +- .../windowing/FoldAllWindowFunction.java | 2 +- .../windowing/FoldWindowFunction.java | 2 +- .../windowing/ReduceAllWindowFunction.java | 44 +- .../ReduceApplyAllWindowFunction.java | 54 ++ .../windowing/ReduceApplyWindowFunction.java | 54 ++ .../ReduceIterableAllWindowFunction.java | 46 ++ .../ReduceIterableWindowFunction.java | 46 ++ .../windowing/ReduceWindowFunction.java | 26 +- .../ReduceWindowFunctionWithWindow.java | 44 +- .../functions/windowing/WindowFunction.java | 4 +- .../triggers/ContinuousEventTimeTrigger.java | 8 +- .../ContinuousProcessingTimeTrigger.java | 11 +- .../api/windowing/triggers/CountTrigger.java | 9 +- .../api/windowing/triggers/DeltaTrigger.java | 18 +- .../api/windowing/triggers/Trigger.java | 57 +- .../windowing/AccumulatingKeyedTimePanes.java | 8 +- ...umulatingProcessingTimeWindowOperator.java | 6 +- .../EvictingNonKeyedWindowOperator.java | 2 +- .../windowing/EvictingWindowOperator.java | 119 +++- .../windowing/NonKeyedWindowOperator.java | 73 ++- .../operators/windowing/WindowOperator.java | 569 ++++++++---------- .../flink/streaming/api/DataStreamTest.java | 2 +- .../api/complex/ComplexIntegrationTest.java | 58 +- ...ignedProcessingTimeWindowOperatorTest.java | 12 +- .../windowing/AllWindowTranslationTest.java | 10 +- .../EvictingNonKeyedWindowOperatorTest.java | 32 +- .../windowing/EvictingWindowOperatorTest.java | 131 +++- .../windowing/NonKeyedWindowOperatorTest.java | 46 +- .../windowing/TimeWindowTranslationTest.java | 8 +- .../windowing/WindowOperatorTest.java | 275 ++++++--- .../windowing/WindowTranslationTest.java | 50 +- .../api/scala/AllWindowedStream.scala | 19 +- .../streaming/api/scala/WindowedStream.scala | 19 +- .../api/scala/AllWindowTranslationTest.scala | 41 +- .../api/scala/WindowTranslationTest.scala | 46 +- ...EventTimeAllWindowCheckpointingITCase.java | 54 +- .../EventTimeWindowCheckpointingITCase.java | 116 ++-- .../WindowCheckpointingITCase.java | 35 +- 49 files changed, 1392 insertions(+), 956 deletions(-) create mode 100644 flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/ReduceApplyAllWindowFunction.java create mode 100644 flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/ReduceApplyWindowFunction.java create mode 100644 flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/ReduceIterableAllWindowFunction.java create mode 100644 flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/ReduceIterableWindowFunction.java diff --git a/flink-examples/flink-examples-streaming/src/main/java/org/apache/flink/streaming/examples/ml/IncrementalLearningSkeleton.java b/flink-examples/flink-examples-streaming/src/main/java/org/apache/flink/streaming/examples/ml/IncrementalLearningSkeleton.java index 32cf4304494a9..8f502dd35815e 100644 --- a/flink-examples/flink-examples-streaming/src/main/java/org/apache/flink/streaming/examples/ml/IncrementalLearningSkeleton.java +++ b/flink-examples/flink-examples-streaming/src/main/java/org/apache/flink/streaming/examples/ml/IncrementalLearningSkeleton.java @@ -172,7 +172,7 @@ public long getCurrentWatermark() { /** * Builds up-to-date partial models on new training data. */ - public static class PartialModelBuilder implements AllWindowFunction { + public static class PartialModelBuilder implements AllWindowFunction, Double[], TimeWindow> { private static final long serialVersionUID = 1L; protected Double[] buildPartialModel(Iterable values) { diff --git a/flink-examples/flink-examples-streaming/src/main/java/org/apache/flink/streaming/examples/windowing/GroupedProcessingTimeWindowExample.java b/flink-examples/flink-examples-streaming/src/main/java/org/apache/flink/streaming/examples/windowing/GroupedProcessingTimeWindowExample.java index f08069b310ec9..196b73e0d14bc 100644 --- a/flink-examples/flink-examples-streaming/src/main/java/org/apache/flink/streaming/examples/windowing/GroupedProcessingTimeWindowExample.java +++ b/flink-examples/flink-examples-streaming/src/main/java/org/apache/flink/streaming/examples/windowing/GroupedProcessingTimeWindowExample.java @@ -104,7 +104,7 @@ public Key getKey(Type value) { } } - public static class SummingWindowFunction implements WindowFunction, Tuple2, Long, Window> { + public static class SummingWindowFunction implements WindowFunction>, Tuple2, Long, Window> { @Override public void apply(Long key, Window window, Iterable> values, Collector> out) { diff --git a/flink-examples/flink-examples-streaming/src/main/java/org/apache/flink/streaming/examples/windowing/SessionWindowing.java b/flink-examples/flink-examples-streaming/src/main/java/org/apache/flink/streaming/examples/windowing/SessionWindowing.java index 7938ee408a9bd..b1d95908e55c5 100644 --- a/flink-examples/flink-examples-streaming/src/main/java/org/apache/flink/streaming/examples/windowing/SessionWindowing.java +++ b/flink-examples/flink-examples-streaming/src/main/java/org/apache/flink/streaming/examples/windowing/SessionWindowing.java @@ -17,7 +17,10 @@ package org.apache.flink.streaming.examples.windowing; +import org.apache.flink.api.common.ExecutionConfig; import org.apache.flink.api.common.state.ValueState; +import org.apache.flink.api.common.state.ValueStateDescriptor; +import org.apache.flink.api.common.typeinfo.BasicTypeInfo; import org.apache.flink.api.java.tuple.Tuple3; import org.apache.flink.streaming.api.TimeCharacteristic; import org.apache.flink.streaming.api.datastream.DataStream; @@ -100,6 +103,10 @@ private static class SessionTrigger implements Trigger stateDesc = new ValueStateDescriptor<>("last-seen", 1L, + BasicTypeInfo.LONG_TYPE_INFO.createSerializer(new ExecutionConfig())); + + public SessionTrigger(Long sessionTimeout) { this.sessionTimeout = sessionTimeout; @@ -108,7 +115,7 @@ public SessionTrigger(Long sessionTimeout) { @Override public TriggerResult onElement(Tuple3 element, long timestamp, GlobalWindow window, TriggerContext ctx) throws Exception { - ValueState lastSeenState = ctx.getKeyValueState("last-seen", 1L); + ValueState lastSeenState = ctx.getPartitionedState(stateDesc); Long lastSeen = lastSeenState.value(); Long timeSinceLastEvent = timestamp - lastSeen; @@ -127,7 +134,7 @@ public TriggerResult onElement(Tuple3 element, long times @Override public TriggerResult onEventTime(long time, GlobalWindow window, TriggerContext ctx) throws Exception { - ValueState lastSeenState = ctx.getKeyValueState("last-seen", 1L); + ValueState lastSeenState = ctx.getPartitionedState(stateDesc); Long lastSeen = lastSeenState.value(); if (time - lastSeen >= sessionTimeout) { diff --git a/flink-examples/flink-examples-streaming/src/main/java/org/apache/flink/streaming/examples/windowing/TopSpeedWindowing.java b/flink-examples/flink-examples-streaming/src/main/java/org/apache/flink/streaming/examples/windowing/TopSpeedWindowing.java index 30eda67c118f8..5a56a40771d57 100644 --- a/flink-examples/flink-examples-streaming/src/main/java/org/apache/flink/streaming/examples/windowing/TopSpeedWindowing.java +++ b/flink-examples/flink-examples-streaming/src/main/java/org/apache/flink/streaming/examples/windowing/TopSpeedWindowing.java @@ -85,7 +85,7 @@ public double getDelta( Tuple4 newDataPoint) { return newDataPoint.f2 - oldDataPoint.f2; } - })) + }, carData.getType().createSerializer(env.getConfig()))) .maxBy(1); if (fileOutput) { diff --git a/flink-examples/flink-examples-streaming/src/main/java/org/apache/flink/streaming/examples/windowing/util/TopSpeedWindowingExampleData.java b/flink-examples/flink-examples-streaming/src/main/java/org/apache/flink/streaming/examples/windowing/util/TopSpeedWindowingExampleData.java index bf636955b48bb..4718b8be9538c 100644 --- a/flink-examples/flink-examples-streaming/src/main/java/org/apache/flink/streaming/examples/windowing/util/TopSpeedWindowingExampleData.java +++ b/flink-examples/flink-examples-streaming/src/main/java/org/apache/flink/streaming/examples/windowing/util/TopSpeedWindowingExampleData.java @@ -192,9 +192,7 @@ public class TopSpeedWindowingExampleData { "(1,95,1973.6111111111115,1424952007664)\n" + "(0,100,1709.7222222222229,1424952006663)\n" + "(0,100,1737.5000000000007,1424952007664)\n" + - "(1,95,1973.6111111111115,1424952007664)\n" + - "(0,100,1791.6666666666674,1424952009664)\n" + - "(1,95,2211.1111111111118,1424952017668)\n"; + "(1,95,1973.6111111111115,1424952007664)\n"; public static final String TOP_CASE_CLASS_SPEEDS = "CarEvent(0,55,15.277777777777777,1424951918630)\n" + @@ -267,9 +265,7 @@ public class TopSpeedWindowingExampleData { "CarEvent(1,95,1973.6111111111115,1424952007664)\n" + "CarEvent(0,100,1709.7222222222229,1424952006663)\n" + "CarEvent(0,100,1737.5000000000007,1424952007664)\n" + - "CarEvent(1,95,1973.6111111111115,1424952007664)\n" + - "CarEvent(0,100,1791.6666666666674,1424952009664)\n" + - "CarEvent(1,95,2211.1111111111118,1424952017668)\n"; + "CarEvent(1,95,1973.6111111111115,1424952007664)\n"; private TopSpeedWindowingExampleData() { } diff --git a/flink-examples/flink-examples-streaming/src/main/scala/org/apache/flink/streaming/scala/examples/windowing/TopSpeedWindowing.scala b/flink-examples/flink-examples-streaming/src/main/scala/org/apache/flink/streaming/scala/examples/windowing/TopSpeedWindowing.scala index f26f32cecc843..c30e654c245b5 100644 --- a/flink-examples/flink-examples-streaming/src/main/scala/org/apache/flink/streaming/scala/examples/windowing/TopSpeedWindowing.scala +++ b/flink-examples/flink-examples-streaming/src/main/scala/org/apache/flink/streaming/scala/examples/windowing/TopSpeedWindowing.scala @@ -72,7 +72,7 @@ object TopSpeedWindowing { .evictor(TimeEvictor.of(Time.of(evictionSec * 1000, TimeUnit.MILLISECONDS))) .trigger(DeltaTrigger.of(triggerMeters, new DeltaFunction[CarEvent] { def getDelta(oldSp: CarEvent, newSp: CarEvent): Double = newSp.distance - oldSp.distance - })) + }, cars.getType().createSerializer(env.getConfig))) // .window(Time.of(evictionSec * 1000, (car : CarEvent) => car.time)) // .every(Delta.of[CarEvent](triggerMeters, // (oldSp,newSp) => newSp.distance-oldSp.distance, CarEvent(0,0,0,0))) diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/AllWindowedStream.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/AllWindowedStream.java index 989e7627446e3..8cef5eaef6742 100644 --- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/AllWindowedStream.java +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/AllWindowedStream.java @@ -21,8 +21,10 @@ import org.apache.flink.api.common.functions.FoldFunction; import org.apache.flink.api.common.functions.Function; import org.apache.flink.api.common.functions.ReduceFunction; +import org.apache.flink.api.common.functions.RichFunction; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.Utils; +import org.apache.flink.api.java.typeutils.GenericTypeInfo; import org.apache.flink.api.java.typeutils.TypeExtractor; import org.apache.flink.streaming.api.TimeCharacteristic; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; @@ -30,8 +32,9 @@ import org.apache.flink.streaming.api.functions.aggregation.ComparableAggregator; import org.apache.flink.streaming.api.functions.aggregation.SumAggregator; import org.apache.flink.streaming.api.functions.windowing.FoldAllWindowFunction; -import org.apache.flink.streaming.api.functions.windowing.ReduceAllWindowFunction; import org.apache.flink.streaming.api.functions.windowing.AllWindowFunction; +import org.apache.flink.streaming.api.functions.windowing.ReduceApplyAllWindowFunction; +import org.apache.flink.streaming.api.functions.windowing.ReduceIterableAllWindowFunction; import org.apache.flink.streaming.api.operators.OneInputStreamOperator; import org.apache.flink.streaming.api.windowing.assigners.WindowAssigner; import org.apache.flink.streaming.api.windowing.evictors.Evictor; @@ -126,6 +129,11 @@ public AllWindowedStream evictor(Evictor evictor) { * @return The data stream that is the result of applying the reduce function to the window. */ public SingleOutputStreamOperator reduce(ReduceFunction function) { + if (function instanceof RichFunction) { + throw new UnsupportedOperationException("ReduceFunction of reduce can not be a RichFunction. " + + "Please use apply(ReduceFunction, WindowFunction) instead."); + } + //clean the closure function = input.getExecutionEnvironment().clean(function); @@ -147,7 +155,7 @@ public AllWindowedStream evictor(Evictor evictor) { operator = new EvictingNonKeyedWindowOperator<>(windowAssigner, windowAssigner.getWindowSerializer(getExecutionEnvironment().getConfig()), new HeapWindowBuffer.Factory(), - new ReduceAllWindowFunction(function), + new ReduceIterableAllWindowFunction(function), trigger, evictor).enableSetProcessingTime(setProcessingTime); @@ -155,7 +163,7 @@ public AllWindowedStream evictor(Evictor evictor) { operator = new NonKeyedWindowOperator<>(windowAssigner, windowAssigner.getWindowSerializer(getExecutionEnvironment().getConfig()), new PreAggregatingHeapWindowBuffer.Factory<>(function), - new ReduceAllWindowFunction(function), + new ReduceIterableAllWindowFunction(function), trigger).enableSetProcessingTime(setProcessingTime); } @@ -205,10 +213,11 @@ public AllWindowedStream evictor(Evictor evictor) { * @param function The window function. * @return The data stream that is the result of applying the window function to the window. */ - public SingleOutputStreamOperator apply(AllWindowFunction function) { - TypeInformation inType = input.getType(); + public SingleOutputStreamOperator apply(AllWindowFunction, R, W> function) { + @SuppressWarnings("unchecked, rawtypes") + TypeInformation> iterTypeInfo = new GenericTypeInfo<>((Class) Iterable.class); TypeInformation resultType = TypeExtractor.getUnaryOperatorReturnType( - function, AllWindowFunction.class, true, true, inType, null, false); + function, AllWindowFunction.class, true, true, iterTypeInfo, null, false); return apply(function, resultType); } @@ -224,7 +233,7 @@ public AllWindowedStream evictor(Evictor evictor) { * @param function The window function. * @return The data stream that is the result of applying the window function to the window. */ - public SingleOutputStreamOperator apply(AllWindowFunction function, TypeInformation resultType) { + public SingleOutputStreamOperator apply(AllWindowFunction, R, W> function, TypeInformation resultType) { //clean the closure function = input.getExecutionEnvironment().clean(function); @@ -297,6 +306,10 @@ public AllWindowedStream evictor(Evictor evictor) { * @return The data stream that is the result of applying the window function to the window. */ public SingleOutputStreamOperator apply(ReduceFunction preAggregator, AllWindowFunction function, TypeInformation resultType) { + if (preAggregator instanceof RichFunction) { + throw new UnsupportedOperationException("Pre-aggregator of apply can not be a RichFunction."); + } + //clean the closures function = input.getExecutionEnvironment().clean(function); preAggregator = input.getExecutionEnvironment().clean(preAggregator); @@ -314,16 +327,16 @@ public AllWindowedStream evictor(Evictor evictor) { operator = new EvictingNonKeyedWindowOperator<>(windowAssigner, windowAssigner.getWindowSerializer(getExecutionEnvironment().getConfig()), new HeapWindowBuffer.Factory(), - function, + new ReduceApplyAllWindowFunction<>(preAggregator, function), trigger, evictor).enableSetProcessingTime(setProcessingTime); } else { operator = new NonKeyedWindowOperator<>(windowAssigner, - windowAssigner.getWindowSerializer(getExecutionEnvironment().getConfig()), - new PreAggregatingHeapWindowBuffer.Factory<>(preAggregator), - function, - trigger).enableSetProcessingTime(setProcessingTime); + windowAssigner.getWindowSerializer(getExecutionEnvironment().getConfig()), + new PreAggregatingHeapWindowBuffer.Factory<>(preAggregator), + new ReduceApplyAllWindowFunction<>(preAggregator, function), + trigger).enableSetProcessingTime(setProcessingTime); } return input.transform(opName, resultType, operator).setParallelism(1); diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/CoGroupedStreams.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/CoGroupedStreams.java index d1da783500770..39030152d4259 100644 --- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/CoGroupedStreams.java +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/CoGroupedStreams.java @@ -545,7 +545,7 @@ public KEY getKey(TaggedUnion value) throws Exception{ private static class CoGroupWindowFunction extends WrappingFunction> - implements WindowFunction, T, KEY, W> { + implements WindowFunction>, T, KEY, W> { private static final long serialVersionUID = 1L; diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/WindowedStream.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/WindowedStream.java index 9dbee30a62387..d64248feacdee 100644 --- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/WindowedStream.java +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/WindowedStream.java @@ -21,9 +21,13 @@ import org.apache.flink.api.common.functions.FoldFunction; import org.apache.flink.api.common.functions.Function; import org.apache.flink.api.common.functions.ReduceFunction; +import org.apache.flink.api.common.functions.RichFunction; +import org.apache.flink.api.common.state.ListStateDescriptor; +import org.apache.flink.api.common.state.ReducingStateDescriptor; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.Utils; import org.apache.flink.api.java.functions.KeySelector; +import org.apache.flink.api.java.typeutils.GenericTypeInfo; import org.apache.flink.api.java.typeutils.TypeExtractor; import org.apache.flink.streaming.api.TimeCharacteristic; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; @@ -31,8 +35,10 @@ import org.apache.flink.streaming.api.functions.aggregation.ComparableAggregator; import org.apache.flink.streaming.api.functions.aggregation.SumAggregator; import org.apache.flink.streaming.api.functions.windowing.FoldWindowFunction; -import org.apache.flink.streaming.api.functions.windowing.WindowFunction; +import org.apache.flink.streaming.api.functions.windowing.ReduceApplyWindowFunction; +import org.apache.flink.streaming.api.functions.windowing.ReduceIterableWindowFunction; import org.apache.flink.streaming.api.functions.windowing.ReduceWindowFunction; +import org.apache.flink.streaming.api.functions.windowing.WindowFunction; import org.apache.flink.streaming.api.operators.OneInputStreamOperator; import org.apache.flink.streaming.api.windowing.assigners.SlidingTimeWindows; import org.apache.flink.streaming.api.windowing.assigners.TumblingTimeWindows; @@ -46,8 +52,8 @@ import org.apache.flink.streaming.runtime.operators.windowing.AggregatingProcessingTimeWindowOperator; import org.apache.flink.streaming.runtime.operators.windowing.EvictingWindowOperator; import org.apache.flink.streaming.runtime.operators.windowing.WindowOperator; -import org.apache.flink.streaming.runtime.operators.windowing.buffers.HeapWindowBuffer; -import org.apache.flink.streaming.runtime.operators.windowing.buffers.PreAggregatingHeapWindowBuffer; +import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; +import org.apache.flink.streaming.runtime.streamrecord.StreamRecordSerializer; /** * A {@code WindowedStream} represents a data stream where elements are grouped by @@ -136,7 +142,13 @@ public WindowedStream evictor(Evictor evictor) { * @param function The reduce function. * @return The data stream that is the result of applying the reduce function to the window. */ + @SuppressWarnings("unchecked") public SingleOutputStreamOperator reduce(ReduceFunction function) { + if (function instanceof RichFunction) { + throw new UnsupportedOperationException("ReduceFunction of reduce can not be a RichFunction. " + + "Please use apply(ReduceFunction, WindowFunction) instead."); + } + //clean the closure function = input.getExecutionEnvironment().clean(function); @@ -156,23 +168,30 @@ public WindowedStream evictor(Evictor evictor) { boolean setProcessingTime = input.getExecutionEnvironment().getStreamTimeCharacteristic() == TimeCharacteristic.ProcessingTime; if (evictor != null) { + ListStateDescriptor> stateDesc = new ListStateDescriptor<>("window-contents", + new StreamRecordSerializer<>(input.getType().createSerializer(getExecutionEnvironment().getConfig()))); + operator = new EvictingWindowOperator<>(windowAssigner, - windowAssigner.getWindowSerializer(getExecutionEnvironment().getConfig()), - keySel, - input.getKeyType().createSerializer(getExecutionEnvironment().getConfig()), - new HeapWindowBuffer.Factory(), - new ReduceWindowFunction(function), - trigger, - evictor).enableSetProcessingTime(setProcessingTime); + windowAssigner.getWindowSerializer(getExecutionEnvironment().getConfig()), + keySel, + input.getKeyType().createSerializer(getExecutionEnvironment().getConfig()), + stateDesc, + new ReduceIterableWindowFunction(function), + trigger, + evictor).enableSetProcessingTime(setProcessingTime); } else { + ReducingStateDescriptor stateDesc = new ReducingStateDescriptor<>("window-contents", + function, + input.getType().createSerializer(getExecutionEnvironment().getConfig())); + operator = new WindowOperator<>(windowAssigner, - windowAssigner.getWindowSerializer(getExecutionEnvironment().getConfig()), - keySel, - input.getKeyType().createSerializer(getExecutionEnvironment().getConfig()), - new PreAggregatingHeapWindowBuffer.Factory<>(function), - new ReduceWindowFunction(function), - trigger).enableSetProcessingTime(setProcessingTime); + windowAssigner.getWindowSerializer(getExecutionEnvironment().getConfig()), + keySel, + input.getKeyType().createSerializer(getExecutionEnvironment().getConfig()), + stateDesc, + new ReduceWindowFunction(), + trigger).enableSetProcessingTime(setProcessingTime); } return input.transform(opName, input.getType(), operator); @@ -222,10 +241,11 @@ public WindowedStream evictor(Evictor evictor) { * @param function The window function. * @return The data stream that is the result of applying the window function to the window. */ - public SingleOutputStreamOperator apply(WindowFunction function) { - TypeInformation inType = input.getType(); + public SingleOutputStreamOperator apply(WindowFunction, R, K, W> function) { + @SuppressWarnings("unchecked, rawtypes") + TypeInformation> iterTypeInfo = new GenericTypeInfo<>((Class) Iterable.class); TypeInformation resultType = TypeExtractor.getUnaryOperatorReturnType( - function, WindowFunction.class, true, true, inType, null, false); + function, WindowFunction.class, true, true, iterTypeInfo, null, false); return apply(function, resultType); } @@ -243,7 +263,8 @@ public WindowedStream evictor(Evictor evictor) { * @param resultType Type information for the result type of the window function * @return The data stream that is the result of applying the window function to the window. */ - public SingleOutputStreamOperator apply(WindowFunction function, TypeInformation resultType) { + public SingleOutputStreamOperator apply(WindowFunction, R, K, W> function, TypeInformation resultType) { + //clean the closure function = input.getExecutionEnvironment().clean(function); @@ -259,26 +280,33 @@ public WindowedStream evictor(Evictor evictor) { String opName = "TriggerWindow(" + windowAssigner + ", " + trigger + ", " + udfName + ")"; KeySelector keySel = input.getKeySelector(); - WindowOperator operator; + WindowOperator, R, W> operator; boolean setProcessingTime = input.getExecutionEnvironment().getStreamTimeCharacteristic() == TimeCharacteristic.ProcessingTime; + if (evictor != null) { + ListStateDescriptor> stateDesc = new ListStateDescriptor<>("window-contents", + new StreamRecordSerializer<>(input.getType().createSerializer(getExecutionEnvironment().getConfig()))); + operator = new EvictingWindowOperator<>(windowAssigner, windowAssigner.getWindowSerializer(getExecutionEnvironment().getConfig()), keySel, input.getKeyType().createSerializer(getExecutionEnvironment().getConfig()), - new HeapWindowBuffer.Factory(), + stateDesc, function, trigger, evictor).enableSetProcessingTime(setProcessingTime); } else { + ListStateDescriptor stateDesc = new ListStateDescriptor<>("window-contents", + input.getType().createSerializer(getExecutionEnvironment().getConfig())); + operator = new WindowOperator<>(windowAssigner, windowAssigner.getWindowSerializer(getExecutionEnvironment().getConfig()), keySel, input.getKeyType().createSerializer(getExecutionEnvironment().getConfig()), - new HeapWindowBuffer.Factory(), + stateDesc, function, trigger).enableSetProcessingTime(setProcessingTime); } @@ -294,17 +322,17 @@ public WindowedStream evictor(Evictor evictor) { *

* Arriving data is pre-aggregated using the given pre-aggregation reducer. * - * @param preAggregator The reduce function that is used for pre-aggregation + * @param reduceFunction The reduce function that is used for pre-aggregation * @param function The window function. * @return The data stream that is the result of applying the window function to the window. */ - public SingleOutputStreamOperator apply(ReduceFunction preAggregator, WindowFunction function) { + public SingleOutputStreamOperator apply(ReduceFunction reduceFunction, WindowFunction function) { TypeInformation inType = input.getType(); TypeInformation resultType = TypeExtractor.getUnaryOperatorReturnType( function, WindowFunction.class, true, true, inType, null, false); - return apply(preAggregator, function, resultType); + return apply(reduceFunction, function, resultType); } /** @@ -315,15 +343,19 @@ public WindowedStream evictor(Evictor evictor) { *

* Arriving data is pre-aggregated using the given pre-aggregation reducer. * - * @param preAggregator The reduce function that is used for pre-aggregation + * @param reduceFunction The reduce function that is used for pre-aggregation * @param function The window function. * @param resultType Type information for the result type of the window function * @return The data stream that is the result of applying the window function to the window. */ - public SingleOutputStreamOperator apply(ReduceFunction preAggregator, WindowFunction function, TypeInformation resultType) { + public SingleOutputStreamOperator apply(ReduceFunction reduceFunction, WindowFunction function, TypeInformation resultType) { + if (reduceFunction instanceof RichFunction) { + throw new UnsupportedOperationException("Pre-aggregator of apply can not be a RichFunction."); + } + //clean the closures function = input.getExecutionEnvironment().clean(function); - preAggregator = input.getExecutionEnvironment().clean(preAggregator); + reduceFunction = input.getExecutionEnvironment().clean(reduceFunction); String callLocation = Utils.getCallLocationName(); String udfName = "WindowApply at " + callLocation; @@ -336,21 +368,29 @@ public WindowedStream evictor(Evictor evictor) { boolean setProcessingTime = input.getExecutionEnvironment().getStreamTimeCharacteristic() == TimeCharacteristic.ProcessingTime; if (evictor != null) { + + ListStateDescriptor> stateDesc = new ListStateDescriptor<>("window-contents", + new StreamRecordSerializer<>(input.getType().createSerializer(getExecutionEnvironment().getConfig()))); + operator = new EvictingWindowOperator<>(windowAssigner, - windowAssigner.getWindowSerializer(getExecutionEnvironment().getConfig()), - keySel, - input.getKeyType().createSerializer(getExecutionEnvironment().getConfig()), - new HeapWindowBuffer.Factory(), - function, - trigger, - evictor).enableSetProcessingTime(setProcessingTime); + windowAssigner.getWindowSerializer(getExecutionEnvironment().getConfig()), + keySel, + input.getKeyType().createSerializer(getExecutionEnvironment().getConfig()), + stateDesc, + new ReduceApplyWindowFunction<>(reduceFunction, function), + trigger, + evictor).enableSetProcessingTime(setProcessingTime); } else { + ReducingStateDescriptor stateDesc = new ReducingStateDescriptor<>("window-contents", + reduceFunction, + input.getType().createSerializer(getExecutionEnvironment().getConfig())); + operator = new WindowOperator<>(windowAssigner, windowAssigner.getWindowSerializer(getExecutionEnvironment().getConfig()), keySel, input.getKeyType().createSerializer(getExecutionEnvironment().getConfig()), - new PreAggregatingHeapWindowBuffer.Factory<>(preAggregator), + stateDesc, function, trigger).enableSetProcessingTime(setProcessingTime); } @@ -587,7 +627,7 @@ public WindowedStream evictor(Evictor evictor) { } else if (function instanceof WindowFunction) { @SuppressWarnings("unchecked") - WindowFunction wf = (WindowFunction) function; + WindowFunction, R, K, TimeWindow> wf = (WindowFunction, R, K, TimeWindow>) function; OneInputStreamOperator op = new AccumulatingProcessingTimeWindowOperator<>( wf, input.getKeySelector(), @@ -619,7 +659,7 @@ else if (function instanceof WindowFunction) { } else if (function instanceof WindowFunction) { @SuppressWarnings("unchecked") - WindowFunction wf = (WindowFunction) function; + WindowFunction, R, K, TimeWindow> wf = (WindowFunction, R, K, TimeWindow>) function; OneInputStreamOperator op = new AccumulatingProcessingTimeWindowOperator<>( wf, input.getKeySelector(), diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/aggregation/AggregationFunction.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/aggregation/AggregationFunction.java index ed39103510a8b..fe711a54bf3f2 100644 --- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/aggregation/AggregationFunction.java +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/aggregation/AggregationFunction.java @@ -17,9 +17,9 @@ package org.apache.flink.streaming.api.functions.aggregation; -import org.apache.flink.api.common.functions.RichReduceFunction; +import org.apache.flink.api.common.functions.ReduceFunction; -public abstract class AggregationFunction extends RichReduceFunction { +public abstract class AggregationFunction implements ReduceFunction { private static final long serialVersionUID = 1L; public enum AggregationType { diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/AllWindowFunction.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/AllWindowFunction.java index 1d544364eaad6..b66bac6af99ea 100644 --- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/AllWindowFunction.java +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/AllWindowFunction.java @@ -30,7 +30,7 @@ * @param The type of the input value. * @param The type of the output value. */ -public interface AllWindowFunction extends Function, Serializable { +public interface AllWindowFunction extends Function, Serializable { /** * Evaluates the window and outputs none or several elements. @@ -41,5 +41,5 @@ public interface AllWindowFunction extends Function, * * @throws Exception The function may throw exceptions to fail the program and trigger recovery. */ - void apply(W window, Iterable values, Collector out) throws Exception; + void apply(W window, IN values, Collector out) throws Exception; } diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/FoldAllWindowFunction.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/FoldAllWindowFunction.java index af32f9be46404..46f9b3c092a8f 100644 --- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/FoldAllWindowFunction.java +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/FoldAllWindowFunction.java @@ -36,7 +36,7 @@ public class FoldAllWindowFunction extends WrappingFunction> - implements AllWindowFunction, OutputTypeConfigurable { + implements AllWindowFunction, R, W>, OutputTypeConfigurable { private static final long serialVersionUID = 1L; private byte[] serializedInitialValue; diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/FoldWindowFunction.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/FoldWindowFunction.java index b1eb3cd2a06b9..db6d1bbff2241 100644 --- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/FoldWindowFunction.java +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/FoldWindowFunction.java @@ -36,7 +36,7 @@ public class FoldWindowFunction extends WrappingFunction> - implements WindowFunction, OutputTypeConfigurable { + implements WindowFunction, R, K, W>, OutputTypeConfigurable { private static final long serialVersionUID = 1L; private byte[] serializedInitialValue; diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/ReduceAllWindowFunction.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/ReduceAllWindowFunction.java index 24855a5bef233..76b095bc87ac0 100644 --- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/ReduceAllWindowFunction.java +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/ReduceAllWindowFunction.java @@ -17,54 +17,14 @@ */ package org.apache.flink.streaming.api.functions.windowing; -import org.apache.flink.api.common.functions.ReduceFunction; -import org.apache.flink.api.common.functions.RuntimeContext; -import org.apache.flink.api.common.functions.util.FunctionUtils; -import org.apache.flink.configuration.Configuration; import org.apache.flink.streaming.api.windowing.windows.Window; import org.apache.flink.util.Collector; public class ReduceAllWindowFunction extends RichAllWindowFunction { private static final long serialVersionUID = 1L; - private final ReduceFunction reduceFunction; - - public ReduceAllWindowFunction(ReduceFunction reduceFunction) { - this.reduceFunction = reduceFunction; - } - - @Override - public void setRuntimeContext(RuntimeContext ctx) { - super.setRuntimeContext(ctx); - FunctionUtils.setFunctionRuntimeContext(reduceFunction, ctx); - } - - @Override - public void open(Configuration parameters) throws Exception { - super.open(parameters); - FunctionUtils.openFunction(reduceFunction, parameters); - } - @Override - public void close() throws Exception { - super.close(); - FunctionUtils.closeFunction(reduceFunction); - } - - @Override - public void apply(W window, Iterable values, Collector out) throws Exception { - T result = null; - - for (T v: values) { - if (result == null) { - result = v; - } else { - result = reduceFunction.reduce(result, v); - } - } - - if (result != null) { - out.collect(result); - } + public void apply(W window, T input, Collector out) throws Exception { + out.collect(input); } } diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/ReduceApplyAllWindowFunction.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/ReduceApplyAllWindowFunction.java new file mode 100644 index 0000000000000..f9fb771602e7e --- /dev/null +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/ReduceApplyAllWindowFunction.java @@ -0,0 +1,54 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.flink.streaming.api.functions.windowing; + +import org.apache.flink.api.common.functions.ReduceFunction; +import org.apache.flink.api.java.operators.translation.WrappingFunction; +import org.apache.flink.streaming.api.windowing.windows.Window; +import org.apache.flink.util.Collector; + +public class ReduceApplyAllWindowFunction + extends WrappingFunction> + implements AllWindowFunction, R, W> { + + private static final long serialVersionUID = 1L; + + private final ReduceFunction reduceFunction; + private final AllWindowFunction windowFunction; + + public ReduceApplyAllWindowFunction(ReduceFunction reduceFunction, + AllWindowFunction windowFunction) { + super(windowFunction); + this.reduceFunction = reduceFunction; + this.windowFunction = windowFunction; + } + + @Override + public void apply(W window, Iterable input, Collector out) throws Exception { + + T curr = null; + for (T val: input) { + if (curr == null) { + curr = val; + } else { + curr = reduceFunction.reduce(curr, val); + } + } + windowFunction.apply(window, curr, out); + } +} diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/ReduceApplyWindowFunction.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/ReduceApplyWindowFunction.java new file mode 100644 index 0000000000000..bf52e9b04f292 --- /dev/null +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/ReduceApplyWindowFunction.java @@ -0,0 +1,54 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.flink.streaming.api.functions.windowing; + +import org.apache.flink.api.common.functions.ReduceFunction; +import org.apache.flink.api.java.operators.translation.WrappingFunction; +import org.apache.flink.streaming.api.windowing.windows.Window; +import org.apache.flink.util.Collector; + +public class ReduceApplyWindowFunction + extends WrappingFunction> + implements WindowFunction, R, K, W> { + + private static final long serialVersionUID = 1L; + + private final ReduceFunction reduceFunction; + private final WindowFunction windowFunction; + + public ReduceApplyWindowFunction(ReduceFunction reduceFunction, + WindowFunction windowFunction) { + super(windowFunction); + this.reduceFunction = reduceFunction; + this.windowFunction = windowFunction; + } + + @Override + public void apply(K k, W window, Iterable input, Collector out) throws Exception { + + T curr = null; + for (T val: input) { + if (curr == null) { + curr = val; + } else { + curr = reduceFunction.reduce(curr, val); + } + } + windowFunction.apply(k, window, curr, out); + } +} diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/ReduceIterableAllWindowFunction.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/ReduceIterableAllWindowFunction.java new file mode 100644 index 0000000000000..2283fe77d8157 --- /dev/null +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/ReduceIterableAllWindowFunction.java @@ -0,0 +1,46 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.flink.streaming.api.functions.windowing; + +import org.apache.flink.api.common.functions.ReduceFunction; +import org.apache.flink.streaming.api.windowing.windows.Window; +import org.apache.flink.util.Collector; + +public class ReduceIterableAllWindowFunction implements AllWindowFunction, T, W> { + private static final long serialVersionUID = 1L; + + private final ReduceFunction reduceFunction; + + public ReduceIterableAllWindowFunction(ReduceFunction reduceFunction) { + this.reduceFunction = reduceFunction; + } + + @Override + public void apply(W window, Iterable input, Collector out) throws Exception { + + T curr = null; + for (T val: input) { + if (curr == null) { + curr = val; + } else { + curr = reduceFunction.reduce(curr, val); + } + } + out.collect(curr); + } +} diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/ReduceIterableWindowFunction.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/ReduceIterableWindowFunction.java new file mode 100644 index 0000000000000..063cee43f273a --- /dev/null +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/ReduceIterableWindowFunction.java @@ -0,0 +1,46 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.flink.streaming.api.functions.windowing; + +import org.apache.flink.api.common.functions.ReduceFunction; +import org.apache.flink.streaming.api.windowing.windows.Window; +import org.apache.flink.util.Collector; + +public class ReduceIterableWindowFunction implements WindowFunction, T, K, W> { + private static final long serialVersionUID = 1L; + + private final ReduceFunction reduceFunction; + + public ReduceIterableWindowFunction(ReduceFunction reduceFunction) { + this.reduceFunction = reduceFunction; + } + + @Override + public void apply(K k, W window, Iterable input, Collector out) throws Exception { + + T curr = null; + for (T val: input) { + if (curr == null) { + curr = val; + } else { + curr = reduceFunction.reduce(curr, val); + } + } + out.collect(curr); + } +} diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/ReduceWindowFunction.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/ReduceWindowFunction.java index edd8a34f40e02..8be4553b274cf 100644 --- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/ReduceWindowFunction.java +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/ReduceWindowFunction.java @@ -17,34 +17,14 @@ */ package org.apache.flink.streaming.api.functions.windowing; -import org.apache.flink.api.common.functions.ReduceFunction; -import org.apache.flink.api.java.operators.translation.WrappingFunction; import org.apache.flink.streaming.api.windowing.windows.Window; import org.apache.flink.util.Collector; -public class ReduceWindowFunction - extends WrappingFunction> - implements WindowFunction { +public class ReduceWindowFunction implements WindowFunction { private static final long serialVersionUID = 1L; - public ReduceWindowFunction(ReduceFunction reduceFunction) { - super(reduceFunction); - } - @Override - public void apply(K k, W window, Iterable values, Collector out) throws Exception { - T result = null; - - for (T v: values) { - if (result == null) { - result = v; - } else { - result = wrappedFunction.reduce(result, v); - } - } - - if (result != null) { - out.collect(result); - } + public void apply(K k, W window, T input, Collector out) throws Exception { + out.collect(input); } } diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/ReduceWindowFunctionWithWindow.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/ReduceWindowFunctionWithWindow.java index 6a472b178246f..fe42cd3000791 100644 --- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/ReduceWindowFunctionWithWindow.java +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/ReduceWindowFunctionWithWindow.java @@ -17,55 +17,15 @@ */ package org.apache.flink.streaming.api.functions.windowing; -import org.apache.flink.api.common.functions.ReduceFunction; -import org.apache.flink.api.common.functions.RuntimeContext; -import org.apache.flink.api.common.functions.util.FunctionUtils; import org.apache.flink.api.java.tuple.Tuple2; -import org.apache.flink.configuration.Configuration; import org.apache.flink.streaming.api.windowing.windows.Window; import org.apache.flink.util.Collector; public class ReduceWindowFunctionWithWindow extends RichWindowFunction, K, W> { private static final long serialVersionUID = 1L; - private final ReduceFunction reduceFunction; - - public ReduceWindowFunctionWithWindow(ReduceFunction reduceFunction) { - this.reduceFunction = reduceFunction; - } - - @Override - public void setRuntimeContext(RuntimeContext ctx) { - super.setRuntimeContext(ctx); - FunctionUtils.setFunctionRuntimeContext(reduceFunction, ctx); - } - - @Override - public void open(Configuration parameters) throws Exception { - super.open(parameters); - FunctionUtils.openFunction(reduceFunction, parameters); - } - @Override - public void close() throws Exception { - super.close(); - FunctionUtils.closeFunction(reduceFunction); - } - - @Override - public void apply(K k, W window, Iterable values, Collector> out) throws Exception { - T result = null; - - for (T v: values) { - if (result == null) { - result = v; - } else { - result = reduceFunction.reduce(result, v); - } - } - - if (result != null) { - out.collect(Tuple2.of(window, result)); - } + public void apply(K k, W window, T input, Collector> out) throws Exception { + out.collect(Tuple2.of(window, input)); } } diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/WindowFunction.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/WindowFunction.java index eda12c04e737f..204d6a59699dc 100644 --- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/WindowFunction.java +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/WindowFunction.java @@ -38,10 +38,10 @@ public interface WindowFunction extends Function * * @param key The key for which this window is evaluated. * @param window The window that is being evaluated. - * @param values The elements in the window being evaluated. + * @param input The elements in the window being evaluated. * @param out A collector for emitting elements. * * @throws Exception The function may throw exceptions to fail the program and trigger recovery. */ - void apply(KEY key, W window, Iterable values, Collector out) throws Exception; + void apply(KEY key, W window, IN input, Collector out) throws Exception; } diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/triggers/ContinuousEventTimeTrigger.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/triggers/ContinuousEventTimeTrigger.java index b653be38f4aed..17818afb7c17a 100644 --- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/triggers/ContinuousEventTimeTrigger.java +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/triggers/ContinuousEventTimeTrigger.java @@ -18,7 +18,10 @@ package org.apache.flink.streaming.api.windowing.triggers; import com.google.common.annotations.VisibleForTesting; +import org.apache.flink.api.common.ExecutionConfig; import org.apache.flink.api.common.state.ValueState; +import org.apache.flink.api.common.state.ValueStateDescriptor; +import org.apache.flink.api.common.typeinfo.BasicTypeInfo; import org.apache.flink.streaming.api.windowing.time.Time; import org.apache.flink.streaming.api.windowing.windows.Window; @@ -35,6 +38,9 @@ public class ContinuousEventTimeTrigger implements Trigger stateDesc = new ValueStateDescriptor<>("first", true, + BasicTypeInfo.BOOLEAN_TYPE_INFO.createSerializer(new ExecutionConfig())); + private ContinuousEventTimeTrigger(long interval) { this.interval = interval; } @@ -42,7 +48,7 @@ private ContinuousEventTimeTrigger(long interval) { @Override public TriggerResult onElement(Object element, long timestamp, W window, TriggerContext ctx) throws Exception { - ValueState first = ctx.getKeyValueState("first", true); + ValueState first = ctx.getPartitionedState(stateDesc); if (first.value()) { long start = timestamp - (timestamp % interval); diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/triggers/ContinuousProcessingTimeTrigger.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/triggers/ContinuousProcessingTimeTrigger.java index 7f3e7ec0ec17c..20a2274abbe4f 100644 --- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/triggers/ContinuousProcessingTimeTrigger.java +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/triggers/ContinuousProcessingTimeTrigger.java @@ -18,7 +18,10 @@ package org.apache.flink.streaming.api.windowing.triggers; import com.google.common.annotations.VisibleForTesting; +import org.apache.flink.api.common.ExecutionConfig; import org.apache.flink.api.common.state.ValueState; +import org.apache.flink.api.common.state.ValueStateDescriptor; +import org.apache.flink.api.common.typeinfo.BasicTypeInfo; import org.apache.flink.streaming.api.windowing.time.Time; import org.apache.flink.streaming.api.windowing.windows.Window; @@ -33,6 +36,10 @@ public class ContinuousProcessingTimeTrigger implements Trigge private final long interval; + private final ValueStateDescriptor stateDesc = new ValueStateDescriptor<>("fire-timestamp", 0L, + BasicTypeInfo.LONG_TYPE_INFO.createSerializer(new ExecutionConfig())); + + private ContinuousProcessingTimeTrigger(long interval) { this.interval = interval; } @@ -41,7 +48,7 @@ private ContinuousProcessingTimeTrigger(long interval) { public TriggerResult onElement(Object element, long timestamp, W window, TriggerContext ctx) throws Exception { long currentTime = System.currentTimeMillis(); - ValueState fireState = ctx.getKeyValueState("fire-timestamp", 0L); + ValueState fireState = ctx.getPartitionedState(stateDesc); long nextFireTimestamp = fireState.value(); if (nextFireTimestamp == 0) { @@ -70,7 +77,7 @@ public TriggerResult onEventTime(long time, W window, TriggerContext ctx) throws @Override public TriggerResult onProcessingTime(long time, W window, TriggerContext ctx) throws Exception { - ValueState fireState = ctx.getKeyValueState("fire-timestamp", 0L); + ValueState fireState = ctx.getPartitionedState(stateDesc); long nextFireTimestamp = fireState.value(); // only fire if an element didn't already fire diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/triggers/CountTrigger.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/triggers/CountTrigger.java index d101fe1e96dca..e8742d5714268 100644 --- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/triggers/CountTrigger.java +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/triggers/CountTrigger.java @@ -17,7 +17,10 @@ */ package org.apache.flink.streaming.api.windowing.triggers; +import org.apache.flink.api.common.ExecutionConfig; import org.apache.flink.api.common.state.ValueState; +import org.apache.flink.api.common.state.ValueStateDescriptor; +import org.apache.flink.api.common.typeinfo.BasicTypeInfo; import org.apache.flink.streaming.api.windowing.windows.Window; import java.io.IOException; @@ -32,13 +35,17 @@ public class CountTrigger implements Trigger { private final long maxCount; + private final ValueStateDescriptor stateDesc = new ValueStateDescriptor<>("count", 0L, + BasicTypeInfo.LONG_TYPE_INFO.createSerializer(new ExecutionConfig())); + + private CountTrigger(long maxCount) { this.maxCount = maxCount; } @Override public TriggerResult onElement(Object element, long timestamp, W window, TriggerContext ctx) throws IOException { - ValueState count = ctx.getKeyValueState("count", 0L); + ValueState count = ctx.getPartitionedState(stateDesc); long currentCount = count.value() + 1; count.update(currentCount); if (currentCount >= maxCount) { diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/triggers/DeltaTrigger.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/triggers/DeltaTrigger.java index 37c8a45c39bb8..60ada88554297 100644 --- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/triggers/DeltaTrigger.java +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/triggers/DeltaTrigger.java @@ -18,11 +18,11 @@ package org.apache.flink.streaming.api.windowing.triggers; import org.apache.flink.api.common.state.ValueState; +import org.apache.flink.api.common.state.ValueStateDescriptor; +import org.apache.flink.api.common.typeutils.TypeSerializer; import org.apache.flink.streaming.api.functions.windowing.delta.DeltaFunction; import org.apache.flink.streaming.api.windowing.windows.Window; -import java.io.Serializable; - /** * A {@link Trigger} that fires based on a {@link DeltaFunction} and a threshold. * @@ -33,20 +33,23 @@ * * @param The type of {@link Window Windows} on which this trigger can operate. */ -public class DeltaTrigger implements Trigger { +public class DeltaTrigger implements Trigger { private static final long serialVersionUID = 1L; private final DeltaFunction deltaFunction; private final double threshold; + private final ValueStateDescriptor stateDesc; - private DeltaTrigger(double threshold, DeltaFunction deltaFunction) { + private DeltaTrigger(double threshold, DeltaFunction deltaFunction, TypeSerializer stateSerializer) { this.deltaFunction = deltaFunction; this.threshold = threshold; + stateDesc = new ValueStateDescriptor<>("last-element", null, stateSerializer); + } @Override public TriggerResult onElement(T element, long timestamp, W window, TriggerContext ctx) throws Exception { - ValueState lastElementState = ctx.getKeyValueState("last-element", null); + ValueState lastElementState = ctx.getPartitionedState(stateDesc); if (lastElementState.value() == null) { lastElementState.update(element); return TriggerResult.CONTINUE; @@ -78,11 +81,12 @@ public String toString() { * * @param threshold The threshold at which to trigger. * @param deltaFunction The delta function to use + * @param stateSerializer TypeSerializer for the data elements. * * @param The type of elements on which this trigger can operate. * @param The type of {@link Window Windows} on which this trigger can operate. */ - public static DeltaTrigger of(double threshold, DeltaFunction deltaFunction) { - return new DeltaTrigger<>(threshold, deltaFunction); + public static DeltaTrigger of(double threshold, DeltaFunction deltaFunction, TypeSerializer stateSerializer) { + return new DeltaTrigger<>(threshold, deltaFunction, stateSerializer); } } diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/triggers/Trigger.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/triggers/Trigger.java index 56f133a60e465..8d61bfc2992ab 100644 --- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/triggers/Trigger.java +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/triggers/Trigger.java @@ -17,7 +17,10 @@ */ package org.apache.flink.streaming.api.windowing.triggers; +import org.apache.flink.api.common.state.State; +import org.apache.flink.api.common.state.StateDescriptor; import org.apache.flink.api.common.state.ValueState; +import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.streaming.api.windowing.windows.Window; import java.io.Serializable; @@ -142,20 +145,62 @@ interface TriggerContext { * Register an event-time callback. When the current watermark passes the specified * time {@link #onEventTime(long, Window, TriggerContext)} is called with the time specified here. * - * @see org.apache.flink.streaming.api.watermark.Watermark - * * @param time The watermark at which to invoke {@link #onEventTime(long, Window, TriggerContext)} + * @see org.apache.flink.streaming.api.watermark.Watermark */ void registerEventTimeTimer(long time); /** - * Retrieves an {@link ValueState} object that can be used to interact with + * Retrieves an {@link State} object that can be used to interact with + * fault-tolerant state that is scoped to the window and key of the current + * trigger invocation. + * + * @param stateDescriptor The StateDescriptor that contains the name and type of the + * state that is being accessed. + * @param The type of the state. + * @return The partitioned state object. + * @throws UnsupportedOperationException Thrown, if no partitioned state is available for the + * function (function is not part os a KeyedStream). + */ + S getPartitionedState(StateDescriptor stateDescriptor); + + /** + * Retrieves a {@link ValueState} object that can be used to interact with * fault-tolerant state that is scoped to the window and key of the current * trigger invocation. * - * @param name A unique key for the state. - * @param defaultState The default value of the state. + * @param name The name of the key/value state. + * @param stateType The class of the type that is stored in the state. Used to generate + * serializers for managed memory and checkpointing. + * @param defaultState The default state value, returned when the state is accessed and + * no value has yet been set for the key. May be null. + * + * @param The type of the state. + * @return The partitioned state object. + * @throws UnsupportedOperationException Thrown, if no partitioned state is available for the + * function (function is not part os a KeyedStream). + */ + @Deprecated + ValueState getKeyValueState(String name, Class stateType, S defaultState); + + + /** + * Retrieves a {@link ValueState} object that can be used to interact with + * fault-tolerant state that is scoped to the window and key of the current + * trigger invocation. + * + * @param name The name of the key/value state. + * @param stateType The type information for the type that is stored in the state. + * Used to create serializers for managed memory and checkpoints. + * @param defaultState The default state value, returned when the state is accessed and + * no value has yet been set for the key. May be null. + * + * @param The type of the state. + * @return The partitioned state object. + * @throws UnsupportedOperationException Thrown, if no partitioned state is available for the + * function (function is not part os a KeyedStream). */ - ValueState getKeyValueState(final String name, final S defaultState); + @Deprecated + ValueState getKeyValueState(String name, TypeInformation stateType, S defaultState); } } diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/AccumulatingKeyedTimePanes.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/AccumulatingKeyedTimePanes.java index e15de8e047b5d..30c40bbd9b951 100644 --- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/AccumulatingKeyedTimePanes.java +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/AccumulatingKeyedTimePanes.java @@ -35,7 +35,7 @@ public class AccumulatingKeyedTimePanes extends AbstractKeyed private final KeyMap.LazyFactory> listFactory = getListFactory(); - private final WindowFunction function; + private final WindowFunction, Result, Key, Window> function; /** * IMPORTANT: This value needs to start at one, so it is fresher than the value that new entries have (zero) */ @@ -43,7 +43,7 @@ public class AccumulatingKeyedTimePanes extends AbstractKeyed // ------------------------------------------------------------------------ - public AccumulatingKeyedTimePanes(KeySelector keySelector, WindowFunction function) { + public AccumulatingKeyedTimePanes(KeySelector keySelector, WindowFunction, Result, Key, Window> function) { this.keySelector = keySelector; this.function = function; } @@ -85,7 +85,7 @@ public void evaluateWindow(Collector out, TimeWindow window, static final class WindowFunctionTraversal implements KeyMap.TraversalEvaluator> { - private final WindowFunction function; + private final WindowFunction, Result, Key, Window> function; private final UnionIterator unionIterator; @@ -98,7 +98,7 @@ static final class WindowFunctionTraversal implements KeyMap. private Key currentKey; - WindowFunctionTraversal(WindowFunction function, TimeWindow window, + WindowFunctionTraversal(WindowFunction, Result, Key, Window> function, TimeWindow window, Collector out, AbstractStreamOperator contextOperator) { this.function = function; this.out = out; diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/AccumulatingProcessingTimeWindowOperator.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/AccumulatingProcessingTimeWindowOperator.java index 7a7d04ced01db..da64df880619a 100644 --- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/AccumulatingProcessingTimeWindowOperator.java +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/AccumulatingProcessingTimeWindowOperator.java @@ -32,13 +32,13 @@ public class AccumulatingProcessingTimeWindowOperator - extends AbstractAlignedProcessingTimeWindowOperator, WindowFunction> { + extends AbstractAlignedProcessingTimeWindowOperator, WindowFunction, OUT, KEY, TimeWindow>> { private static final long serialVersionUID = 7305948082830843475L; public AccumulatingProcessingTimeWindowOperator( - WindowFunction function, + WindowFunction, OUT, KEY, TimeWindow> function, KeySelector keySelector, TypeSerializer keySerializer, TypeSerializer valueSerializer, @@ -52,7 +52,7 @@ public AccumulatingProcessingTimeWindowOperator( @Override protected AccumulatingKeyedTimePanes createPanes(KeySelector keySelector, Function function) { @SuppressWarnings("unchecked") - WindowFunction windowFunction = (WindowFunction) function; + WindowFunction, OUT, KEY, Window> windowFunction = (WindowFunction, OUT, KEY, Window>) function; return new AccumulatingKeyedTimePanes<>(keySelector, windowFunction); } diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/EvictingNonKeyedWindowOperator.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/EvictingNonKeyedWindowOperator.java index 1bb451a4ce94b..73972e60dd763 100644 --- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/EvictingNonKeyedWindowOperator.java +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/EvictingNonKeyedWindowOperator.java @@ -47,7 +47,7 @@ public class EvictingNonKeyedWindowOperator extends N public EvictingNonKeyedWindowOperator(WindowAssigner windowAssigner, TypeSerializer windowSerializer, WindowBufferFactory> windowBufferFactory, - AllWindowFunction windowFunction, + AllWindowFunction, OUT, W> windowFunction, Trigger trigger, Evictor evictor) { super(windowAssigner, windowSerializer, windowBufferFactory, windowFunction, trigger); diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/EvictingWindowOperator.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/EvictingWindowOperator.java index ad43812491c89..f163de1806765 100644 --- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/EvictingWindowOperator.java +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/EvictingWindowOperator.java @@ -18,15 +18,22 @@ package org.apache.flink.streaming.runtime.operators.windowing; import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Function; +import com.google.common.collect.FluentIterable; +import com.google.common.collect.Iterables; +import org.apache.flink.api.common.state.ListState; +import org.apache.flink.api.common.state.MergingState; +import org.apache.flink.api.common.state.StateDescriptor; import org.apache.flink.api.common.typeutils.TypeSerializer; import org.apache.flink.api.java.functions.KeySelector; import org.apache.flink.streaming.api.functions.windowing.WindowFunction; import org.apache.flink.streaming.api.windowing.assigners.WindowAssigner; -import org.apache.flink.streaming.runtime.operators.windowing.buffers.EvictingWindowBuffer; -import org.apache.flink.streaming.runtime.operators.windowing.buffers.WindowBufferFactory; import org.apache.flink.streaming.api.windowing.evictors.Evictor; import org.apache.flink.streaming.api.windowing.triggers.Trigger; import org.apache.flink.streaming.api.windowing.windows.Window; +import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; + +import java.util.Collection; import static java.util.Objects.requireNonNull; @@ -42,42 +49,97 @@ * @param The type of elements emitted by the {@code WindowFunction}. * @param The type of {@code Window} that the {@code WindowAssigner} assigns. */ -public class EvictingWindowOperator extends WindowOperator { +public class EvictingWindowOperator extends WindowOperator, OUT, W> { private static final long serialVersionUID = 1L; private final Evictor evictor; + private final StateDescriptor>> windowStateDescriptor; + public EvictingWindowOperator(WindowAssigner windowAssigner, - TypeSerializer windowSerializer, - KeySelector keySelector, - TypeSerializer keySerializer, - WindowBufferFactory> windowBufferFactory, - WindowFunction windowFunction, - Trigger trigger, - Evictor evictor) { - super(windowAssigner, windowSerializer, keySelector, keySerializer, windowBufferFactory, windowFunction, trigger); + TypeSerializer windowSerializer, + KeySelector keySelector, + TypeSerializer keySerializer, + StateDescriptor>> windowStateDescriptor, + WindowFunction, OUT, K, W> windowFunction, + Trigger trigger, + Evictor evictor) { + super(windowAssigner, windowSerializer, keySelector, keySerializer, null, windowFunction, trigger); this.evictor = requireNonNull(evictor); + this.windowStateDescriptor = windowStateDescriptor; } + @Override - @SuppressWarnings("unchecked, rawtypes") - protected void emitWindow(Context context) throws Exception { - timestampedCollector.setTimestamp(context.window.maxTimestamp()); - EvictingWindowBuffer windowBuffer = (EvictingWindowBuffer) context.windowBuffer; - - int toEvict = 0; - if (windowBuffer.size() > 0) { - // need some type trickery here... - toEvict = evictor.evict((Iterable) windowBuffer.getElements(), windowBuffer.size(), context.window); + @SuppressWarnings("unchecked") + public final void processElement(StreamRecord element) throws Exception { + Collection elementWindows = windowAssigner.assignWindows(element.getValue(), element.getTimestamp()); + + K key = (K) getStateBackend().getCurrentKey(); + + for (W window: elementWindows) { + + ListState> windowState = getPartitionedState(window, windowSerializer, + windowStateDescriptor); + + windowState.add(element); + + context.key = key; + context.window = window; + Trigger.TriggerResult triggerResult = context.onElement(element); + + processTriggerResult(triggerResult, key, window); } + } - windowBuffer.removeElements(toEvict); + @Override + @SuppressWarnings("unchecked,rawtypes") + protected void processTriggerResult(Trigger.TriggerResult triggerResult, K key, W window) throws Exception { + if (!triggerResult.isFire() && !triggerResult.isPurge()) { + // do nothing + return; + } - userFunction.apply(context.key, - context.window, - context.windowBuffer.getUnpackedElements(), - timestampedCollector); + if (triggerResult.isFire()) { + timestampedCollector.setTimestamp(window.maxTimestamp()); + + setKeyContext(key); + + ListState> windowState = getPartitionedState(window, windowSerializer, + windowStateDescriptor); + + Iterable> contents = windowState.get(); + + // Work around type system restrictions... + int toEvict = evictor.evict((Iterable) contents, Iterables.size(contents), context.window); + + FluentIterable projectedContents = FluentIterable + .from(contents) + .skip(toEvict) + .transform(new Function, IN>() { + @Override + public IN apply(StreamRecord input) { + return input.getValue(); + } + }); + userFunction.apply(context.key, context.window, projectedContents, timestampedCollector); + + if (triggerResult.isPurge()) { + windowState.clear(); + } else { + // we have to clear the state and set the elements that remain after eviction + windowState.clear(); + for (StreamRecord rec: FluentIterable.from(contents).skip(toEvict)) { + windowState.add(rec); + } + } + } else if (triggerResult.isPurge()) { + setKeyContext(key); + ListState> windowState = getPartitionedState(window, windowSerializer, + windowStateDescriptor); + windowState.clear(); + } } @Override @@ -95,4 +157,11 @@ public EvictingWindowOperator enableSetProcessingTime(boolean set public Evictor getEvictor() { return evictor; } + + @Override + @VisibleForTesting + @SuppressWarnings("unchecked, rawtypes") + public StateDescriptor>> getStateDescriptor() { + return (StateDescriptor>>) windowStateDescriptor; + } } diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/NonKeyedWindowOperator.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/NonKeyedWindowOperator.java index cce56573e5966..291a019ddd139 100644 --- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/NonKeyedWindowOperator.java +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/NonKeyedWindowOperator.java @@ -19,10 +19,14 @@ import com.google.common.annotations.VisibleForTesting; import org.apache.flink.api.common.ExecutionConfig; +import org.apache.flink.api.common.state.State; +import org.apache.flink.api.common.state.StateDescriptor; import org.apache.flink.api.common.state.ValueState; +import org.apache.flink.api.common.state.ValueStateDescriptor; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.common.typeutils.TypeSerializer; import org.apache.flink.api.java.typeutils.InputTypeConfigurable; +import org.apache.flink.api.java.typeutils.TypeExtractor; import org.apache.flink.core.memory.DataInputView; import org.apache.flink.runtime.state.AbstractStateBackend; import org.apache.flink.runtime.state.StateHandle; @@ -69,7 +73,7 @@ * @param The type of {@code Window} that the {@code WindowAssigner} assigns. */ public class NonKeyedWindowOperator - extends AbstractUdfStreamOperator> + extends AbstractUdfStreamOperator, OUT, W>> implements OneInputStreamOperator, Triggerable, InputTypeConfigurable { private static final long serialVersionUID = 1L; @@ -145,7 +149,7 @@ public class NonKeyedWindowOperator public NonKeyedWindowOperator(WindowAssigner windowAssigner, TypeSerializer windowSerializer, WindowBufferFactory> windowBufferFactory, - AllWindowFunction windowFunction, + AllWindowFunction, OUT, W> windowFunction, Trigger trigger) { super(windowFunction); @@ -413,29 +417,72 @@ protected void writeToState(AbstractStateBackend.CheckpointStateOutputView out) } } - @SuppressWarnings("unchecked") - public ValueState getKeyValueState(final String name, final S defaultState) { - return new ValueState() { + @Override + public ValueState getKeyValueState(String name, + Class stateType, + S defaultState) { + requireNonNull(stateType, "The state type class must not be null"); + + TypeInformation typeInfo; + try { + typeInfo = TypeExtractor.getForClass(stateType); + } + catch (Exception e) { + throw new RuntimeException("Cannot analyze type '" + stateType.getName() + + "' from the class alone, due to generic type parameters. " + + "Please specify the TypeInformation directly.", e); + } + + return getKeyValueState(name, typeInfo, defaultState); + } + + @Override + public ValueState getKeyValueState(String name, + TypeInformation stateType, + S defaultState) { + + requireNonNull(name, "The name of the state must not be null"); + requireNonNull(stateType, "The state type information must not be null"); + + ValueStateDescriptor stateDesc = new ValueStateDescriptor<>(name, defaultState, stateType.createSerializer(getExecutionConfig())); + return getPartitionedState(stateDesc); + } + + @Override + @SuppressWarnings("rawtypes, unchecked") + public S getPartitionedState(final StateDescriptor stateDescriptor) { + if (!(stateDescriptor instanceof ValueStateDescriptor)) { + throw new UnsupportedOperationException("NonKeyedWindowOperator Triggers only " + + "support ValueState."); + } + @SuppressWarnings("unchecked") + final ValueStateDescriptor valueStateDescriptor = (ValueStateDescriptor) stateDescriptor; + ValueState valueState = new ValueState() { @Override - public S value() throws IOException { - Serializable value = state.get(name); + public Object value() throws IOException { + Object value = state.get(stateDescriptor.getName()); if (value == null) { - state.put(name, defaultState); - value = defaultState; + value = valueStateDescriptor.getDefaultValue(); + state.put(stateDescriptor.getName(), (Serializable) value); } - return (S) value; + return value; } @Override - public void update(S value) throws IOException { - state.put(name, value); + public void update(Object value) throws IOException { + if (!(value instanceof Serializable)) { + throw new UnsupportedOperationException( + "Value state of NonKeyedWindowOperator must be serializable."); + } + state.put(stateDescriptor.getName(), (Serializable) value); } @Override public void clear() { - state.remove(name); + state.remove(stateDescriptor.getName()); } }; + return (S) valueState; } @Override diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/WindowOperator.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/WindowOperator.java index 46170b5176a97..5109dae4d15d8 100644 --- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/WindowOperator.java +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/WindowOperator.java @@ -19,11 +19,16 @@ import com.google.common.annotations.VisibleForTesting; import org.apache.flink.api.common.ExecutionConfig; +import org.apache.flink.api.common.state.MergingState; +import org.apache.flink.api.common.state.State; +import org.apache.flink.api.common.state.StateDescriptor; import org.apache.flink.api.common.state.ValueState; +import org.apache.flink.api.common.state.ValueStateDescriptor; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.common.typeutils.TypeSerializer; import org.apache.flink.api.java.functions.KeySelector; import org.apache.flink.api.java.typeutils.InputTypeConfigurable; +import org.apache.flink.api.java.typeutils.TypeExtractor; import org.apache.flink.core.memory.DataInputView; import org.apache.flink.runtime.state.AbstractStateBackend; import org.apache.flink.runtime.state.StateHandle; @@ -37,25 +42,18 @@ import org.apache.flink.streaming.api.windowing.triggers.Trigger; import org.apache.flink.streaming.api.windowing.windows.Window; import org.apache.flink.streaming.runtime.operators.Triggerable; -import org.apache.flink.streaming.runtime.operators.windowing.buffers.WindowBuffer; import org.apache.flink.streaming.runtime.operators.windowing.buffers.WindowBufferFactory; -import org.apache.flink.streaming.runtime.streamrecord.MultiplexingStreamRecordSerializer; import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; import org.apache.flink.streaming.runtime.tasks.StreamTaskState; -import org.apache.flink.util.InstantiationUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; import java.io.ObjectInputStream; import java.io.Serializable; -import java.util.ArrayList; import java.util.Collection; -import java.util.HashMap; import java.util.HashSet; -import java.util.Iterator; -import java.util.List; -import java.util.Map; +import java.util.PriorityQueue; import java.util.Set; import static java.util.Objects.requireNonNull; @@ -86,8 +84,8 @@ * @param The type of elements emitted by the {@code WindowFunction}. * @param The type of {@code Window} that the {@code WindowAssigner} assigns. */ -public class WindowOperator - extends AbstractUdfStreamOperator> +public class WindowOperator + extends AbstractUdfStreamOperator> implements OneInputStreamOperator, Triggerable, InputTypeConfigurable { private static final long serialVersionUID = 1L; @@ -98,51 +96,41 @@ public class WindowOperator // Configuration values and user functions // ------------------------------------------------------------------------ - private final WindowAssigner windowAssigner; + protected final WindowAssigner windowAssigner; - private final KeySelector keySelector; + protected final KeySelector keySelector; - private final Trigger trigger; + protected final Trigger trigger; - private final WindowBufferFactory> windowBufferFactory; + protected final StateDescriptor> windowStateDescriptor; /** * If this is true. The current processing time is set as the timestamp of incoming elements. * This for use with a {@link org.apache.flink.streaming.api.windowing.evictors.TimeEvictor} * if eviction should happen based on processing time. */ - private boolean setProcessingTime = false; + protected boolean setProcessingTime = false; /** * This is used to copy the incoming element because it can be put into several window * buffers. */ - private TypeSerializer inputSerializer; + protected TypeSerializer inputSerializer; /** * For serializing the key in checkpoints. */ - private final TypeSerializer keySerializer; + protected final TypeSerializer keySerializer; /** * For serializing the window in checkpoints. */ - private final TypeSerializer windowSerializer; + protected final TypeSerializer windowSerializer; // ------------------------------------------------------------------------ // State that is not checkpointed // ------------------------------------------------------------------------ - /** - * Processing time timers that are currently in-flight. - */ - private transient Map> processingTimeTimers; - - /** - * Current waiting watermark callbacks. - */ - private transient Map> watermarkTimers; - /** * This is given to the {@code WindowFunction} for emitting elements with a given timestamp. */ @@ -154,15 +142,23 @@ public class WindowOperator */ protected transient long currentWatermark = -1L; + protected transient Context context = new Context(null, null); + // ------------------------------------------------------------------------ // State that needs to be checkpointed // ------------------------------------------------------------------------ /** - * The windows (panes) that are currently in-flight. Each pane has a {@code WindowBuffer} - * and a {@code TriggerContext} that stores the {@code Trigger} for that pane. + * Processing time timers that are currently in-flight. + */ + protected transient Set> processingTimeTimers; + protected transient PriorityQueue> processingTimeTimersQueue; + + /** + * Current waiting watermark callbacks. */ - protected transient Map> windows; + protected transient Set> watermarkTimers; + protected transient PriorityQueue> watermarkTimersQueue; /** * Creates a new {@code WindowOperator} based on the given policies and user functions. @@ -171,8 +167,8 @@ public WindowOperator(WindowAssigner windowAssigner, TypeSerializer windowSerializer, KeySelector keySelector, TypeSerializer keySerializer, - WindowBufferFactory> windowBufferFactory, - WindowFunction windowFunction, + StateDescriptor> windowStateDescriptor, + WindowFunction windowFunction, Trigger trigger) { super(windowFunction); @@ -182,7 +178,7 @@ public WindowOperator(WindowAssigner windowAssigner, this.keySelector = requireNonNull(keySelector); this.keySerializer = requireNonNull(keySerializer); - this.windowBufferFactory = requireNonNull(windowBufferFactory); + this.windowStateDescriptor = windowStateDescriptor; this.trigger = requireNonNull(trigger); setChainingStrategy(ChainingStrategy.ALWAYS); @@ -209,162 +205,100 @@ public final void open() throws Exception { throw new IllegalStateException("Input serializer was not set."); } - windowBufferFactory.setRuntimeContext(getRuntimeContext()); - windowBufferFactory.open(getUserFunctionParameters()); - - // these could already be initialized from restoreState() if (watermarkTimers == null) { - watermarkTimers = new HashMap<>(); + watermarkTimers = new HashSet<>(); + watermarkTimersQueue = new PriorityQueue<>(100); } if (processingTimeTimers == null) { - processingTimeTimers = new HashMap<>(); + processingTimeTimers = new HashSet<>(); + processingTimeTimersQueue = new PriorityQueue<>(100); } - if (windows == null) { - windows = new HashMap<>(); - } - - // re-register timers that this window context had set - for (Map.Entry> entry: windows.entrySet()) { - Map keyWindows = entry.getValue(); - for (Context context: keyWindows.values()) { - if (context.processingTimeTimer > 0) { - Set triggers = processingTimeTimers.get(context.processingTimeTimer); - if (triggers == null) { - getRuntimeContext().registerTimer(context.processingTimeTimer, WindowOperator.this); - triggers = new HashSet<>(); - processingTimeTimers.put(context.processingTimeTimer, triggers); - } - triggers.add(context); - } - if (context.watermarkTimer > 0) { - Set triggers = watermarkTimers.get(context.watermarkTimer); - if (triggers == null) { - triggers = new HashSet<>(); - watermarkTimers.put(context.watermarkTimer, triggers); - } - triggers.add(context); - } - } - } + context = new Context(null, null); } @Override public final void close() throws Exception { super.close(); - // emit the elements that we still keep - for (Map.Entry> entry: windows.entrySet()) { - Map keyWindows = entry.getValue(); - for (Context window: keyWindows.values()) { - emitWindow(window); - } - } - windows.clear(); - windowBufferFactory.close(); } @Override @SuppressWarnings("unchecked") - public final void processElement(StreamRecord element) throws Exception { + public void processElement(StreamRecord element) throws Exception { if (setProcessingTime) { element.replace(element.getValue(), System.currentTimeMillis()); } Collection elementWindows = windowAssigner.assignWindows(element.getValue(), element.getTimestamp()); - K key = keySelector.getKey(element.getValue()); - - Map keyWindows = windows.get(key); - if (keyWindows == null) { - keyWindows = new HashMap<>(); - windows.put(key, keyWindows); - } + K key = (K) getStateBackend().getCurrentKey(); for (W window: elementWindows) { - Context context = keyWindows.get(window); - if (context == null) { - WindowBuffer windowBuffer = windowBufferFactory.create(); - context = new Context(key, window, windowBuffer); - keyWindows.put(window, context); - } - - context.windowBuffer.storeElement(element); - Trigger.TriggerResult triggerResult = context.onElement(element); - processTriggerResult(triggerResult, key, window); - } - } - protected void emitWindow(Context context) throws Exception { - timestampedCollector.setTimestamp(context.window.maxTimestamp()); + MergingState windowState = getPartitionedState(window, windowSerializer, + windowStateDescriptor); + windowState.add(element.getValue()); - if (context.windowBuffer.size() > 0) { - setKeyContextElement1(context.windowBuffer.getElements().iterator().next()); + context.key = key; + context.window = window; + Trigger.TriggerResult triggerResult = context.onElement(element); - userFunction.apply(context.key, - context.window, - context.windowBuffer.getUnpackedElements(), - timestampedCollector); + processTriggerResult(triggerResult, key, window); } } - private void processTriggerResult(Trigger.TriggerResult triggerResult, K key, W window) throws Exception { + protected void processTriggerResult(Trigger.TriggerResult triggerResult, K key, W window) throws Exception { if (!triggerResult.isFire() && !triggerResult.isPurge()) { // do nothing return; } - Context context; - Map keyWindows = windows.get(key); - if (keyWindows == null) { - LOG.debug("Window {} for key {} already gone.", window, key); - return; - } - - if (triggerResult.isPurge()) { - context = keyWindows.remove(window); - if (keyWindows.isEmpty()) { - windows.remove(key); - } - } else { - context = keyWindows.get(window); - } - if (context == null) { - LOG.debug("Window {} for key {} already gone.", window, key); - return; - } if (triggerResult.isFire()) { - emitWindow(context); + timestampedCollector.setTimestamp(window.maxTimestamp()); + + setKeyContext(key); + + MergingState windowState = getPartitionedState(window, windowSerializer, + windowStateDescriptor); + + ACC contents = windowState.get(); + + userFunction.apply(context.key, context.window, contents, timestampedCollector); + + if (triggerResult.isPurge()) { + windowState.clear(); + } + } else if (triggerResult.isPurge()) { + setKeyContext(key); + MergingState windowState = getPartitionedState(window, windowSerializer, + windowStateDescriptor); + windowState.clear(); } } @Override public final void processWatermark(Watermark mark) throws Exception { - List> toTrigger = new ArrayList<>(); - Iterator>> it = watermarkTimers.entrySet().iterator(); + boolean fire; - while (it.hasNext()) { - Map.Entry> triggers = it.next(); - if (triggers.getKey() <= mark.getTimestamp()) { - toTrigger.add(triggers.getValue()); - it.remove(); - } - } + do { + Timer timer = watermarkTimersQueue.peek(); + if (timer != null && timer.timestamp <= mark.getTimestamp()) { + fire = true; + + watermarkTimers.remove(timer); + watermarkTimersQueue.remove(); - for (Set ctxs: toTrigger) { - for (Context ctx: ctxs) { - // double check the time. it can happen that the trigger registers a new timer, - // in that case the entry is left in the watermarkTimers set for performance reasons. - // We have to check here whether the entry in the set still reflects the - // currently set timer in the Context. - if (ctx.watermarkTimer <= mark.getTimestamp()) { - Trigger.TriggerResult triggerResult = ctx.onEventTime(ctx.watermarkTimer); - processTriggerResult(triggerResult, ctx.key, ctx.window); - } + context.key = timer.key; + context.window = timer.window; + Trigger.TriggerResult triggerResult = context.onEventTime(mark.getTimestamp()); + processTriggerResult(triggerResult, context.key, context.window); + } else { + fire = false; } - } + } while (fire); output.emitWatermark(mark); @@ -373,206 +307,173 @@ public final void processWatermark(Watermark mark) throws Exception { @Override public final void trigger(long time) throws Exception { - List> toTrigger = new ArrayList<>(); + boolean fire; - Iterator>> it = processingTimeTimers.entrySet().iterator(); + do { + Timer timer = processingTimeTimersQueue.peek(); + if (timer != null && timer.timestamp <= time) { + fire = true; - while (it.hasNext()) { - Map.Entry> triggers = it.next(); - if (triggers.getKey() <= time) { - toTrigger.add(triggers.getValue()); - it.remove(); - } - } + processingTimeTimers.remove(timer); + processingTimeTimersQueue.remove(); - for (Set ctxs: toTrigger) { - for (Context ctx: ctxs) { - // double check the time. it can happen that the trigger registers a new timer, - // in that case the entry is left in the processingTimeTimers set for - // performance reasons. We have to check here whether the entry in the set still - // reflects the currently set timer in the Context. - if (ctx.processingTimeTimer <= time) { - Trigger.TriggerResult triggerResult = ctx.onProcessingTime(ctx.processingTimeTimer); - processTriggerResult(triggerResult, ctx.key, ctx.window); - } + context.key = timer.key; + context.window = timer.window; + Trigger.TriggerResult triggerResult = context.onProcessingTime(time); + processTriggerResult(triggerResult, context.key, context.window); + } else { + fire = false; } - } + } while (fire); + + // Also check any watermark timers. We might have some in here since + // Context.registerEventTimeTimer sets a trigger if an event-time trigger is registered + // that is already behind the watermark. + processWatermark(new Watermark(currentWatermark)); } /** - * The {@code Context} is responsible for keeping track of the state of one pane. - * - *

- * A pane is the bucket of elements that have the same key (assigned by the - * {@link org.apache.flink.api.java.functions.KeySelector}) and same {@link Window}. An element can - * be in multiple panes of it was assigned to multiple windows by the - * {@link org.apache.flink.streaming.api.windowing.assigners.WindowAssigner}. These panes all - * have their own instance of the {@code Trigger}. + * {@code Context} is a utility for handling {@code Trigger} invocations. It can be reused + * by setting the {@code key} and {@code window} fields. No internal state must be kept in + * the {@code Context} */ protected class Context implements Trigger.TriggerContext { protected K key; protected W window; - protected WindowBuffer windowBuffer; - - protected HashMap state; - - // use these to only allow one timer in flight at a time of each type - // if the trigger registers another timer this value here will be overwritten, - // the timer is not removed from the set of in-flight timers to improve performance. - // When a trigger fires it is just checked against the last timer that was set. - protected long watermarkTimer; - protected long processingTimeTimer; - - public Context(K key, - W window, - WindowBuffer windowBuffer) { + public Context(K key, W window) { this.key = key; this.window = window; - this.windowBuffer = windowBuffer; - state = new HashMap<>(); - - this.watermarkTimer = -1; - this.processingTimeTimer = -1; } - /** - * Constructs a new {@code Context} by reading from a {@link DataInputView} that - * contains a serialized context that we wrote in - * {@link #writeToState(AbstractStateBackend.CheckpointStateOutputView)} - */ - @SuppressWarnings("unchecked") - protected Context(DataInputView in, ClassLoader userClassloader) throws Exception { - this.key = keySerializer.deserialize(in); - this.window = windowSerializer.deserialize(in); - this.watermarkTimer = in.readLong(); - this.processingTimeTimer = in.readLong(); - - int stateSize = in.readInt(); - byte[] stateData = new byte[stateSize]; - in.read(stateData); - state = InstantiationUtil.deserializeObject(stateData, userClassloader); - - this.windowBuffer = windowBufferFactory.create(); - int numElements = in.readInt(); - MultiplexingStreamRecordSerializer recordSerializer = new MultiplexingStreamRecordSerializer<>(inputSerializer); - for (int i = 0; i < numElements; i++) { - windowBuffer.storeElement(recordSerializer.deserialize(in).asRecord()); + @Override + public ValueState getKeyValueState(String name, + Class stateType, + S defaultState) { + requireNonNull(stateType, "The state type class must not be null"); + + TypeInformation typeInfo; + try { + typeInfo = TypeExtractor.getForClass(stateType); } + catch (Exception e) { + throw new RuntimeException("Cannot analyze type '" + stateType.getName() + + "' from the class alone, due to generic type parameters. " + + "Please specify the TypeInformation directly.", e); + } + + return getKeyValueState(name, typeInfo, defaultState); } - /** - * Writes the {@code Context} to the given state checkpoint output. - */ - protected void writeToState(AbstractStateBackend.CheckpointStateOutputView out) throws IOException { - keySerializer.serialize(key, out); - windowSerializer.serialize(window, out); - out.writeLong(watermarkTimer); - out.writeLong(processingTimeTimer); - - byte[] serializedState = InstantiationUtil.serializeObject(state); - out.writeInt(serializedState.length); - out.write(serializedState, 0, serializedState.length); - - MultiplexingStreamRecordSerializer recordSerializer = new MultiplexingStreamRecordSerializer<>(inputSerializer); - out.writeInt(windowBuffer.size()); - for (StreamRecord element: windowBuffer.getElements()) { - recordSerializer.serialize(element, out); - } + @Override + public ValueState getKeyValueState(String name, + TypeInformation stateType, + S defaultState) { + + requireNonNull(name, "The name of the state must not be null"); + requireNonNull(stateType, "The state type information must not be null"); + + ValueStateDescriptor stateDesc = new ValueStateDescriptor<>(name, defaultState, stateType.createSerializer(getExecutionConfig())); + return getPartitionedState(stateDesc); } @SuppressWarnings("unchecked") - public ValueState getKeyValueState(final String name, final S defaultState) { - return new ValueState() { - @Override - public S value() throws IOException { - Serializable value = state.get(name); - if (value == null) { - state.put(name, defaultState); - value = defaultState; - } - return (S) value; - } - - @Override - public void update(S value) throws IOException { - state.put(name, value); - } - - @Override - public void clear() { - state.remove(name); - } - }; + public S getPartitionedState(StateDescriptor stateDescriptor) { + try { + return WindowOperator.this.getPartitionedState(window, windowSerializer, + stateDescriptor); + } catch (Exception e) { + throw new RuntimeException("Could not retrieve state", e); + } } @Override public void registerProcessingTimeTimer(long time) { - if (this.processingTimeTimer == time) { - // we already have set a trigger for that time - return; - } - Set triggers = processingTimeTimers.get(time); - if (triggers == null) { + Timer timer = new Timer<>(time, key, window); + if (processingTimeTimers.add(timer)) { + processingTimeTimersQueue.add(timer); getRuntimeContext().registerTimer(time, WindowOperator.this); - triggers = new HashSet<>(); - processingTimeTimers.put(time, triggers); } - this.processingTimeTimer = time; - triggers.add(this); } @Override public void registerEventTimeTimer(long time) { - if (watermarkTimer == time) { - // we already have set a trigger for that time - return; + Timer timer = new Timer<>(time, key, window); + if (watermarkTimers.add(timer)) { + watermarkTimersQueue.add(timer); } - Set triggers = watermarkTimers.get(time); - if (triggers == null) { - triggers = new HashSet<>(); - watermarkTimers.put(time, triggers); + + if (time <= currentWatermark) { + // immediately schedule a trigger, so that we don't wait for the next + // watermark update to fire the watermark trigger + getRuntimeContext().registerTimer(time, WindowOperator.this); } - this.watermarkTimer = time; - triggers.add(this); } public Trigger.TriggerResult onElement(StreamRecord element) throws Exception { - Trigger.TriggerResult onElementResult = trigger.onElement(element.getValue(), element.getTimestamp(), window, this); - if (watermarkTimer > 0 && watermarkTimer <= currentWatermark) { - // fire now and don't wait for the next watermark update - Trigger.TriggerResult onEventTimeResult = onEventTime(watermarkTimer); - return Trigger.TriggerResult.merge(onElementResult, onEventTimeResult); - } else { - return onElementResult; - } + return trigger.onElement(element.getValue(), element.getTimestamp(), window, this); } public Trigger.TriggerResult onProcessingTime(long time) throws Exception { - if (time == processingTimeTimer) { - processingTimeTimer = -1; - return trigger.onProcessingTime(time, window, this); - } else { - return Trigger.TriggerResult.CONTINUE; - } + return trigger.onProcessingTime(time, window, this); } public Trigger.TriggerResult onEventTime(long time) throws Exception { - if (time == watermarkTimer) { - watermarkTimer = -1; - Trigger.TriggerResult firstTriggerResult = trigger.onEventTime(time, window, this); - - if (watermarkTimer > 0 && watermarkTimer <= currentWatermark) { - // fire now and don't wait for the next watermark update - Trigger.TriggerResult secondTriggerResult = onEventTime(watermarkTimer); - return Trigger.TriggerResult.merge(firstTriggerResult, secondTriggerResult); - } else { - return firstTriggerResult; - } + return trigger.onEventTime(time, window, this); + } + } - } else { - return Trigger.TriggerResult.CONTINUE; + /** + * Internal class for keeping track of in-flight timers. + */ + protected static class Timer implements Comparable> { + protected long timestamp; + protected K key; + protected W window; + + public Timer(long timestamp, K key, W window) { + this.timestamp = timestamp; + this.key = key; + this.window = window; + } + + @Override + public int compareTo(Timer o) { + return Long.compare(this.timestamp, o.timestamp); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; } + if (o == null || getClass() != o.getClass()){ + return false; + } + + Timer timer = (Timer) o; + + return timestamp == timer.timestamp + && key.equals(timer.key) + && window.equals(timer.window); + + } + + @Override + public int hashCode() { + int result = (int) (timestamp ^ (timestamp >>> 32)); + result = 31 * result + key.hashCode(); + result = 31 * result + window.hashCode(); + return result; + } + + @Override + public String toString() { + return "Timer{" + + "timestamp=" + timestamp + + ", key=" + key + + ", window=" + window + + '}'; } } @@ -582,7 +483,7 @@ public Trigger.TriggerResult onEventTime(long time) throws Exception { * {@link org.apache.flink.streaming.api.windowing.evictors.TimeEvictor} with processing * time semantics. */ - public WindowOperator enableSetProcessingTime(boolean setProcessingTime) { + public WindowOperator enableSetProcessingTime(boolean setProcessingTime) { this.setProcessingTime = setProcessingTime; return this; } @@ -595,21 +496,25 @@ public WindowOperator enableSetProcessingTime(boolean setProcessi public StreamTaskState snapshotOperatorState(long checkpointId, long timestamp) throws Exception { StreamTaskState taskState = super.snapshotOperatorState(checkpointId, timestamp); - // we write the panes with the key/value maps into the stream - AbstractStateBackend.CheckpointStateOutputView out = getStateBackend().createCheckpointStateOutputView(checkpointId, timestamp); + AbstractStateBackend.CheckpointStateOutputView out = + getStateBackend().createCheckpointStateOutputView(checkpointId, timestamp); - int numKeys = windows.size(); - out.writeInt(numKeys); + out.writeInt(watermarkTimersQueue.size()); + for (Timer timer : watermarkTimersQueue) { + keySerializer.serialize(timer.key, out); + windowSerializer.serialize(timer.window, out); + out.writeLong(timer.timestamp); + } - for (Map.Entry> keyWindows: windows.entrySet()) { - int numWindows = keyWindows.getValue().size(); - out.writeInt(numWindows); - for (Context context: keyWindows.getValue().values()) { - context.writeToState(out); - } + out.writeInt(processingTimeTimers.size()); + for (Timer timer : processingTimeTimersQueue) { + keySerializer.serialize(timer.key, out); + windowSerializer.serialize(timer.window, out); + out.writeLong(timer.timestamp); } taskState.setOperatorState(out.closeAndGetHandle()); + return taskState; } @@ -623,22 +528,28 @@ public void restoreState(StreamTaskState taskState, long recoveryTimestamp) thro StateHandle inputState = (StateHandle) taskState.getOperatorState(); DataInputView in = inputState.getState(userClassloader); - int numKeys = in.readInt(); - this.windows = new HashMap<>(numKeys); - this.processingTimeTimers = new HashMap<>(); - this.watermarkTimers = new HashMap<>(); - - for (int i = 0; i < numKeys; i++) { - int numWindows = in.readInt(); - for (int j = 0; j < numWindows; j++) { - Context context = new Context(in, userClassloader); - Map keyWindows = windows.get(context.key); - if (keyWindows == null) { - keyWindows = new HashMap<>(numWindows); - windows.put(context.key, keyWindows); - } - keyWindows.put(context.window, context); - } + int numWatermarkTimers = in.readInt(); + watermarkTimers = new HashSet<>(numWatermarkTimers); + watermarkTimersQueue = new PriorityQueue<>(Math.max(numWatermarkTimers, 1)); + for (int i = 0; i < numWatermarkTimers; i++) { + K key = keySerializer.deserialize(in); + W window = windowSerializer.deserialize(in); + long timestamp = in.readLong(); + Timer timer = new Timer<>(timestamp, key, window); + watermarkTimers.add(timer); + watermarkTimersQueue.add(timer); + } + + int numProcessingTimeTimers = in.readInt(); + processingTimeTimers = new HashSet<>(numProcessingTimeTimers); + processingTimeTimersQueue = new PriorityQueue<>(Math.max(numProcessingTimeTimers, 1)); + for (int i = 0; i < numProcessingTimeTimers; i++) { + K key = keySerializer.deserialize(in); + W window = windowSerializer.deserialize(in); + long timestamp = in.readLong(); + Timer timer = new Timer<>(timestamp, key, window); + processingTimeTimers.add(timer); + processingTimeTimersQueue.add(timer); } } @@ -667,7 +578,7 @@ public WindowAssigner getWindowAssigner() { } @VisibleForTesting - public WindowBufferFactory> getWindowBufferFactory() { - return windowBufferFactory; + public StateDescriptor> getStateDescriptor() { + return windowStateDescriptor; } } diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/DataStreamTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/DataStreamTest.java index 475a95df67271..037afe4092a77 100644 --- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/DataStreamTest.java +++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/DataStreamTest.java @@ -518,7 +518,7 @@ public Tuple2 map(Long value) throws Exception { DataStream window = map .windowAll(GlobalWindows.create()) .trigger(PurgingTrigger.of(CountTrigger.of(5))) - .apply(new AllWindowFunction, String, GlobalWindow>() { + .apply(new AllWindowFunction>, String, GlobalWindow>() { @Override public void apply(GlobalWindow window, Iterable> values, diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/complex/ComplexIntegrationTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/complex/ComplexIntegrationTest.java index 020dda37a4189..2dced46ea0bba 100644 --- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/complex/ComplexIntegrationTest.java +++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/complex/ComplexIntegrationTest.java @@ -21,9 +21,13 @@ import org.apache.flink.api.common.functions.FlatMapFunction; import org.apache.flink.api.common.functions.MapFunction; import org.apache.flink.api.common.functions.ReduceFunction; +import org.apache.flink.api.common.typeutils.TypeSerializer; +import org.apache.flink.api.common.typeutils.base.IntSerializer; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.api.java.tuple.Tuple4; import org.apache.flink.api.java.tuple.Tuple5; +import org.apache.flink.api.java.typeutils.runtime.TupleSerializer; +import org.apache.flink.api.java.typeutils.runtime.kryo.KryoSerializer; import org.apache.flink.configuration.Configuration; import org.apache.flink.core.fs.FileSystem; import org.apache.flink.streaming.api.collector.selector.OutputSelector; @@ -212,7 +216,10 @@ public void complexIntegrationTest2() throws Exception { env.execute(); } + // Ignore because the count(10_000) window actually only emits one element during processing + // and all the rest in close() @SuppressWarnings("unchecked") + @Ignore @Test public void complexIntegrationTest3() throws Exception { //Heavy prime factorisation with maps and flatmaps @@ -247,6 +254,7 @@ public void complexIntegrationTest3() throws Exception { DataStream sourceStream31 = env.generateSequence(1, 10000); DataStream sourceStream32 = env.generateSequence(10001, 20000); + sourceStream31.filter(new PrimeFilterFunction()) .windowAll(GlobalWindows.create()) .trigger(PurgingTrigger.of(CountTrigger.of(100))) @@ -257,9 +265,10 @@ public void complexIntegrationTest3() throws Exception { .max(0)) .writeAsText(resultPath1, FileSystem.WriteMode.OVERWRITE); - sourceStream31.flatMap(new DivisorsFlatMapFunction()) - .union(sourceStream32.flatMap(new DivisorsFlatMapFunction())).map(new MapFunction>() { + sourceStream31 + .flatMap(new DivisorsFlatMapFunction()) + .union(sourceStream32.flatMap(new DivisorsFlatMapFunction())) + .map(new MapFunction>() { @Override public Tuple2 map(Long value) throws Exception { @@ -270,42 +279,49 @@ public Tuple2 map(Long value) throws Exception { .window(GlobalWindows.create()) .trigger(PurgingTrigger.of(CountTrigger.of(10_000))) .sum(1) - .filter(new FilterFunction>() { - @Override - public boolean filter(Tuple2 value) throws Exception { - return value.f0 < 100 || value.f0 > 19900; - } - }) - .writeAsText(resultPath2, FileSystem.WriteMode.OVERWRITE); +// .filter(new FilterFunction>() { +// +// @Override +// public boolean filter(Tuple2 value) throws Exception { +// return value.f0 < 100 || value.f0 > 19900; +// } +// }) + .print(); +// .writeAsText(resultPath2, FileSystem.WriteMode.OVERWRITE); env.execute(); } @Test @Ignore + @SuppressWarnings("unchecked, rawtypes") public void complexIntegrationTest4() throws Exception { //Testing mapping and delta-policy windowing with custom class expected1 = "((100,100),0)\n" + "((120,122),5)\n" + "((121,125),6)\n" + "((138,144),9)\n" + - "((139,147),10)\n" + "((156,166),13)\n" + "((157,169),14)\n" + "((174,188),17)\n" + "((175,191),18)\n" + - "((192,210),21)\n" + "((193,213),22)\n" + "((210,232),25)\n" + "((211,235),26)\n" + "((228,254),29)\n" + - "((229,257),30)\n" + "((246,276),33)\n" + "((247,279),34)\n" + "((264,298),37)\n" + "((265,301),38)\n" + - "((282,320),41)\n" + "((283,323),42)\n" + "((300,342),45)\n" + "((301,345),46)\n" + "((318,364),49)\n" + - "((319,367),50)\n" + "((336,386),53)\n" + "((337,389),54)\n" + "((354,408),57)\n" + "((355,411),58)\n" + - "((372,430),61)\n" + "((373,433),62)\n" + "((390,452),65)\n" + "((391,455),66)\n" + "((408,474),69)\n" + - "((409,477),70)\n" + "((426,496),73)\n" + "((427,499),74)\n" + "((444,518),77)\n" + "((445,521),78)\n" + - "((462,540),81)\n" + "((463,543),82)\n" + "((480,562),85)\n" + "((481,565),86)\n" + "((498,584),89)\n" + - "((499,587),90)\n" + "((516,606),93)\n" + "((517,609),94)\n" + "((534,628),97)\n" + "((535,631),98)"; + "((139,147),10)\n" + "((156,166),13)\n" + "((157,169),14)\n" + "((174,188),17)\n" + "((175,191),18)\n" + + "((192,210),21)\n" + "((193,213),22)\n" + "((210,232),25)\n" + "((211,235),26)\n" + "((228,254),29)\n" + + "((229,257),30)\n" + "((246,276),33)\n" + "((247,279),34)\n" + "((264,298),37)\n" + "((265,301),38)\n" + + "((282,320),41)\n" + "((283,323),42)\n" + "((300,342),45)\n" + "((301,345),46)\n" + "((318,364),49)\n" + + "((319,367),50)\n" + "((336,386),53)\n" + "((337,389),54)\n" + "((354,408),57)\n" + "((355,411),58)\n" + + "((372,430),61)\n" + "((373,433),62)\n" + "((390,452),65)\n" + "((391,455),66)\n" + "((408,474),69)\n" + + "((409,477),70)\n" + "((426,496),73)\n" + "((427,499),74)\n" + "((444,518),77)\n" + "((445,521),78)\n" + + "((462,540),81)\n" + "((463,543),82)\n" + "((480,562),85)\n" + "((481,565),86)\n" + "((498,584),89)\n" + + "((499,587),90)\n" + "((516,606),93)\n" + "((517,609),94)\n" + "((534,628),97)\n" + "((535,631),98)"; StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); env.setParallelism(1); + TupleSerializer> deltaSerializer = new TupleSerializer<>((Class) Tuple2.class, + new TypeSerializer[] {new KryoSerializer<>(Rectangle.class, env.getConfig()), + IntSerializer.INSTANCE}); + env.addSource(new RectangleSource()) .global() .map(new RectangleMapFunction()) .windowAll(GlobalWindows.create()) - .trigger(PurgingTrigger.of(DeltaTrigger.of(0.0, new MyDelta()))) + .trigger(PurgingTrigger.of(DeltaTrigger.of(0.0, new MyDelta(), deltaSerializer))) .apply(new MyWindowMapFunction()) .writeAsText(resultPath1, FileSystem.WriteMode.OVERWRITE); @@ -673,7 +689,7 @@ public Tuple2 map(Rectangle value) throws Exception { } } - private static class MyWindowMapFunction implements AllWindowFunction, Tuple2, GlobalWindow> { + private static class MyWindowMapFunction implements AllWindowFunction>, Tuple2, GlobalWindow> { private static final long serialVersionUID = 1L; @Override diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/AccumulatingAlignedProcessingTimeWindowOperatorTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/AccumulatingAlignedProcessingTimeWindowOperatorTest.java index 0e7001c314d49..6601e3ee1eac8 100644 --- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/AccumulatingAlignedProcessingTimeWindowOperatorTest.java +++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/AccumulatingAlignedProcessingTimeWindowOperatorTest.java @@ -66,7 +66,7 @@ public class AccumulatingAlignedProcessingTimeWindowOperatorTest { @SuppressWarnings("unchecked") - private final WindowFunction mockFunction = mock(WindowFunction.class); + private final WindowFunction, String, String, TimeWindow> mockFunction = mock(WindowFunction.class); @SuppressWarnings("unchecked") private final KeySelector mockKeySelector = mock(KeySelector.class); @@ -78,8 +78,8 @@ public Integer getKey(Integer value) { } }; - private final WindowFunction validatingIdentityFunction = - new WindowFunction() + private final WindowFunction, Integer, Integer, TimeWindow> validatingIdentityFunction = + new WindowFunction, Integer, Integer, TimeWindow>() { @Override public void apply(Integer key, @@ -494,7 +494,7 @@ public void testPropagateExceptionsFromClose() { final Object lock = new Object(); final StreamTask mockTask = createMockTaskWithTimer(timerService, lock); - WindowFunction failingFunction = new FailingFunction(100); + WindowFunction, Integer, Integer, TimeWindow> failingFunction = new FailingFunction(100); // the operator has a window time that is so long that it will not fire in this test final long hundredYears = 100L * 365 * 24 * 60 * 60 * 1000; @@ -817,7 +817,7 @@ private void assertInvalidParameter(long windowSize, long windowSlide) { // ------------------------------------------------------------------------ - private static class FailingFunction implements WindowFunction { + private static class FailingFunction implements WindowFunction, Integer, Integer, TimeWindow> { private final int failAfterElements; @@ -845,7 +845,7 @@ public void apply(Integer integer, // ------------------------------------------------------------------------ - private static class StatefulFunction extends RichWindowFunction { + private static class StatefulFunction extends RichWindowFunction, Integer, Integer, TimeWindow> { // we use a concurrent map here even though there is no concurrency, to // get "volatile" style access to entries diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/AllWindowTranslationTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/AllWindowTranslationTest.java index 282c71f44b755..d9ba872db454c 100644 --- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/AllWindowTranslationTest.java +++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/AllWindowTranslationTest.java @@ -17,7 +17,7 @@ */ package org.apache.flink.streaming.runtime.operators.windowing; -import org.apache.flink.api.common.functions.RichReduceFunction; +import org.apache.flink.api.common.functions.ReduceFunction; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.streaming.api.TimeCharacteristic; import org.apache.flink.streaming.api.datastream.DataStream; @@ -77,7 +77,7 @@ public void testEventTime() throws Exception { DataStream> window2 = source .windowAll(TumblingTimeWindows.of(Time.of(1, TimeUnit.SECONDS))) - .apply(new AllWindowFunction, Tuple2, TimeWindow>() { + .apply(new AllWindowFunction>, Tuple2, TimeWindow>() { private static final long serialVersionUID = 1L; @Override @@ -126,7 +126,7 @@ public void testNonEvicting() throws Exception { DataStream> window2 = source .windowAll(TumblingTimeWindows.of(Time.of(1, TimeUnit.SECONDS))) .trigger(CountTrigger.of(100)) - .apply(new AllWindowFunction, Tuple2, TimeWindow>() { + .apply(new AllWindowFunction>, Tuple2, TimeWindow>() { private static final long serialVersionUID = 1L; @Override @@ -177,7 +177,7 @@ public void testEvicting() throws Exception { .windowAll(TumblingTimeWindows.of(Time.of(1, TimeUnit.SECONDS))) .trigger(CountTrigger.of(100)) .evictor(TimeEvictor.of(Time.of(100, TimeUnit.MILLISECONDS))) - .apply(new AllWindowFunction, Tuple2, TimeWindow>() { + .apply(new AllWindowFunction>, Tuple2, TimeWindow>() { private static final long serialVersionUID = 1L; @Override @@ -204,7 +204,7 @@ public void apply( // UDFs // ------------------------------------------------------------------------ - public static class DummyReducer extends RichReduceFunction> { + public static class DummyReducer implements ReduceFunction> { private static final long serialVersionUID = 1L; @Override diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/EvictingNonKeyedWindowOperatorTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/EvictingNonKeyedWindowOperatorTest.java index 39033cc0f3995..571838f714061 100644 --- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/EvictingNonKeyedWindowOperatorTest.java +++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/EvictingNonKeyedWindowOperatorTest.java @@ -22,8 +22,7 @@ import org.apache.flink.api.java.functions.KeySelector; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.api.java.typeutils.TypeInfoParser; -import org.apache.flink.configuration.Configuration; -import org.apache.flink.streaming.api.functions.windowing.ReduceAllWindowFunction; +import org.apache.flink.streaming.api.functions.windowing.ReduceIterableAllWindowFunction; import org.apache.flink.streaming.api.watermark.Watermark; import org.apache.flink.streaming.api.windowing.assigners.GlobalWindows; import org.apache.flink.streaming.api.windowing.evictors.CountEvictor; @@ -56,7 +55,7 @@ public void testCountTrigger() throws Exception { GlobalWindows.create(), new GlobalWindow.Serializer(), new HeapWindowBuffer.Factory>(), - new ReduceAllWindowFunction>(new SumReducer(closeCalled)), + new ReduceIterableAllWindowFunction>(new SumReducer()), CountTrigger.of(WINDOW_SLIDE), CountEvictor.of(WINDOW_SIZE)); @@ -96,10 +95,6 @@ public void testCountTrigger() throws Exception { TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new ResultSortComparator()); testHarness.close(); - - Assert.assertEquals("Close was not called.", 1, closeCalled.get()); - - } // ------------------------------------------------------------------------ @@ -109,32 +104,9 @@ public void testCountTrigger() throws Exception { public static class SumReducer extends RichReduceFunction> { private static final long serialVersionUID = 1L; - private boolean openCalled = false; - - private AtomicInteger closeCalled; - - public SumReducer(AtomicInteger closeCalled) { - this.closeCalled = closeCalled; - } - - @Override - public void open(Configuration parameters) throws Exception { - super.open(parameters); - openCalled = true; - } - - @Override - public void close() throws Exception { - super.close(); - closeCalled.incrementAndGet(); - } - @Override public Tuple2 reduce(Tuple2 value1, Tuple2 value2) throws Exception { - if (!openCalled) { - Assert.fail("Open was not called"); - } return new Tuple2<>(value2.f0, value1.f1 + value2.f1); } } diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/EvictingWindowOperatorTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/EvictingWindowOperatorTest.java index 1821308fa7ce3..2f1dce5567dea 100644 --- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/EvictingWindowOperatorTest.java +++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/EvictingWindowOperatorTest.java @@ -18,22 +18,27 @@ package org.apache.flink.streaming.runtime.operators.windowing; import org.apache.flink.api.common.ExecutionConfig; -import org.apache.flink.api.common.functions.RichReduceFunction; +import org.apache.flink.api.common.functions.ReduceFunction; +import org.apache.flink.api.common.state.ListStateDescriptor; import org.apache.flink.api.common.typeinfo.BasicTypeInfo; +import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.functions.KeySelector; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.api.java.typeutils.TypeInfoParser; import org.apache.flink.configuration.Configuration; +import org.apache.flink.streaming.api.functions.windowing.ReduceIterableWindowFunction; +import org.apache.flink.streaming.api.functions.windowing.RichWindowFunction; import org.apache.flink.streaming.api.watermark.Watermark; import org.apache.flink.streaming.api.windowing.assigners.GlobalWindows; -import org.apache.flink.streaming.runtime.operators.windowing.buffers.HeapWindowBuffer; import org.apache.flink.streaming.api.windowing.evictors.CountEvictor; -import org.apache.flink.streaming.api.functions.windowing.ReduceWindowFunction; import org.apache.flink.streaming.api.windowing.triggers.CountTrigger; import org.apache.flink.streaming.api.windowing.windows.GlobalWindow; +import org.apache.flink.streaming.api.windowing.windows.Window; import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; +import org.apache.flink.streaming.runtime.streamrecord.StreamRecordSerializer; import org.apache.flink.streaming.util.OneInputStreamOperatorTestHarness; import org.apache.flink.streaming.util.TestHarnessUtil; +import org.apache.flink.util.Collector; import org.junit.Assert; import org.junit.Test; @@ -48,27 +53,35 @@ public class EvictingWindowOperatorTest { @Test @SuppressWarnings("unchecked") public void testCountTrigger() throws Exception { - AtomicInteger closeCalled = new AtomicInteger(0); final int WINDOW_SIZE = 4; final int WINDOW_SLIDE = 2; + TypeInformation> inputType = TypeInfoParser.parse("Tuple2"); + + + ListStateDescriptor>> stateDesc = new ListStateDescriptor<>("window-contents", + new StreamRecordSerializer<>(inputType.createSerializer(new ExecutionConfig()))); + + EvictingWindowOperator, Tuple2, GlobalWindow> operator = new EvictingWindowOperator<>( GlobalWindows.create(), new GlobalWindow.Serializer(), new TupleKeySelector(), BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()), - new HeapWindowBuffer.Factory>(), - new ReduceWindowFunction>(new SumReducer(closeCalled)), + stateDesc, + new ReduceIterableWindowFunction>(new SumReducer()), CountTrigger.of(WINDOW_SLIDE), CountEvictor.of(WINDOW_SIZE)); - operator.setInputType(TypeInfoParser.>parse("Tuple2"), new ExecutionConfig()); + operator.setInputType(inputType, new ExecutionConfig()); OneInputStreamOperatorTestHarness, Tuple2> testHarness = new OneInputStreamOperatorTestHarness<>(operator); + testHarness.configureForKeyedStream(new TupleKeySelector(), BasicTypeInfo.STRING_TYPE_INFO); + long initialTime = 0L; ConcurrentLinkedQueue expectedOutput = new ConcurrentLinkedQueue<>(); @@ -105,24 +118,104 @@ public void testCountTrigger() throws Exception { TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new ResultSortComparator()); testHarness.close(); + } - Assert.assertEquals("Close was not called.", 1, closeCalled.get()); + @Test + @SuppressWarnings("unchecked") + public void testCountTriggerWithApply() throws Exception { + AtomicInteger closeCalled = new AtomicInteger(0); + + final int WINDOW_SIZE = 4; + final int WINDOW_SLIDE = 2; + + TypeInformation> inputType = TypeInfoParser.parse("Tuple2"); + + + ListStateDescriptor>> stateDesc = new ListStateDescriptor<>("window-contents", + new StreamRecordSerializer<>(inputType.createSerializer(new ExecutionConfig()))); + + + EvictingWindowOperator, Tuple2, GlobalWindow> operator = new EvictingWindowOperator<>( + GlobalWindows.create(), + new GlobalWindow.Serializer(), + new TupleKeySelector(), + BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()), + stateDesc, + new RichSumReducer(closeCalled), + CountTrigger.of(WINDOW_SLIDE), + CountEvictor.of(WINDOW_SIZE)); + + operator.setInputType(inputType, new ExecutionConfig()); + + + OneInputStreamOperatorTestHarness, Tuple2> testHarness = + new OneInputStreamOperatorTestHarness<>(operator); + + testHarness.configureForKeyedStream(new TupleKeySelector(), BasicTypeInfo.STRING_TYPE_INFO); + + long initialTime = 0L; + ConcurrentLinkedQueue expectedOutput = new ConcurrentLinkedQueue<>(); + + testHarness.open(); + + // The global window actually ignores these timestamps... + + // add elements out-of-order + testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 3000)); + testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 3999)); + + testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), initialTime + 20)); + testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), initialTime)); + testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), initialTime + 999)); + testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 1998)); + testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 1999)); + testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 1000)); + + + + expectedOutput.add(new StreamRecord<>(new Tuple2<>("key2", 2), Long.MAX_VALUE)); + expectedOutput.add(new StreamRecord<>(new Tuple2<>("key2", 4), Long.MAX_VALUE)); + expectedOutput.add(new StreamRecord<>(new Tuple2<>("key1", 2), Long.MAX_VALUE)); + + TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new ResultSortComparator()); + + testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), initialTime + 10999)); + testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 1000)); + + expectedOutput.add(new StreamRecord<>(new Tuple2<>("key1", 4), Long.MAX_VALUE)); + expectedOutput.add(new StreamRecord<>(new Tuple2<>("key2", 4), Long.MAX_VALUE)); + + TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new ResultSortComparator()); + + testHarness.close(); + Assert.assertEquals("Close was not called.", 1, closeCalled.get()); } // ------------------------------------------------------------------------ // UDFs // ------------------------------------------------------------------------ - public static class SumReducer extends RichReduceFunction> { + public static class SumReducer implements ReduceFunction> { + private static final long serialVersionUID = 1L; + + + @Override + public Tuple2 reduce(Tuple2 value1, + Tuple2 value2) throws Exception { + return new Tuple2<>(value2.f0, value1.f1 + value2.f1); + } + } + + public static class RichSumReducer extends RichWindowFunction>, Tuple2, String, W> { private static final long serialVersionUID = 1L; private boolean openCalled = false; - private AtomicInteger closeCalled; + private AtomicInteger closeCalled = new AtomicInteger(0); - public SumReducer(AtomicInteger closeCalled) { + public RichSumReducer(AtomicInteger closeCalled) { this.closeCalled = closeCalled; } @@ -139,13 +232,23 @@ public void close() throws Exception { } @Override - public Tuple2 reduce(Tuple2 value1, - Tuple2 value2) throws Exception { + public void apply(String key, + W window, + Iterable> input, + Collector> out) throws Exception { + if (!openCalled) { Assert.fail("Open was not called"); } - return new Tuple2<>(value2.f0, value1.f1 + value2.f1); + int sum = 0; + + for (Tuple2 t: input) { + sum += t.f1; + } + out.collect(new Tuple2<>(key, sum)); + } + } @SuppressWarnings("unchecked") diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/NonKeyedWindowOperatorTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/NonKeyedWindowOperatorTest.java index 02e032a6be990..c0e6ad4bae724 100644 --- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/NonKeyedWindowOperatorTest.java +++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/NonKeyedWindowOperatorTest.java @@ -18,11 +18,12 @@ package org.apache.flink.streaming.runtime.operators.windowing; import org.apache.flink.api.common.ExecutionConfig; +import org.apache.flink.api.common.functions.ReduceFunction; import org.apache.flink.api.common.functions.RichReduceFunction; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.api.java.typeutils.TypeInfoParser; import org.apache.flink.configuration.Configuration; -import org.apache.flink.streaming.api.functions.windowing.ReduceAllWindowFunction; +import org.apache.flink.streaming.api.functions.windowing.ReduceIterableAllWindowFunction; import org.apache.flink.streaming.api.watermark.Watermark; import org.apache.flink.streaming.api.windowing.assigners.GlobalWindows; import org.apache.flink.streaming.api.windowing.assigners.SlidingTimeWindows; @@ -77,7 +78,7 @@ public void testSlidingEventTimeWindows() throws Exception { SlidingTimeWindows.of(Time.of(WINDOW_SIZE, TimeUnit.SECONDS), Time.of(WINDOW_SLIDE, TimeUnit.SECONDS)), new TimeWindow.Serializer(), windowBufferFactory, - new ReduceAllWindowFunction>(new SumReducer()), + new ReduceIterableAllWindowFunction>(new SumReducer()), EventTimeTrigger.create()); operator.setInputType(TypeInfoParser.>parse("Tuple2"), new ExecutionConfig()); @@ -140,11 +141,6 @@ public void testSlidingEventTimeWindows() throws Exception { TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new ResultSortComparator()); testHarness.close(); - if (windowBufferFactory instanceof PreAggregatingHeapWindowBuffer.Factory) { - Assert.assertEquals("Close was not called.", 2, closeCalled.get()); - } else { - Assert.assertEquals("Close was not called.", 1, closeCalled.get()); - } } @Test @@ -158,7 +154,7 @@ public void testTumblingEventTimeWindows() throws Exception { TumblingTimeWindows.of(Time.of(WINDOW_SIZE, TimeUnit.SECONDS)), new TimeWindow.Serializer(), windowBufferFactory, - new ReduceAllWindowFunction>(new SumReducer()), + new ReduceIterableAllWindowFunction>(new SumReducer()), EventTimeTrigger.create()); operator.setInputType(TypeInfoParser.>parse("Tuple2"), new ExecutionConfig()); @@ -219,11 +215,6 @@ public void testTumblingEventTimeWindows() throws Exception { TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new ResultSortComparator()); testHarness.close(); - if (windowBufferFactory instanceof PreAggregatingHeapWindowBuffer.Factory) { - Assert.assertEquals("Close was not called.", 2, closeCalled.get()); - } else { - Assert.assertEquals("Close was not called.", 1, closeCalled.get()); - } } @Test @@ -237,7 +228,7 @@ public void testContinuousWatermarkTrigger() throws Exception { GlobalWindows.create(), new GlobalWindow.Serializer(), windowBufferFactory, - new ReduceAllWindowFunction>(new SumReducer()), + new ReduceIterableAllWindowFunction>(new SumReducer()), ContinuousEventTimeTrigger.of(Time.of(WINDOW_SIZE, TimeUnit.SECONDS))); operator.setInputType(TypeInfoParser.>parse("Tuple2"), new ExecutionConfig()); @@ -298,11 +289,6 @@ public void testContinuousWatermarkTrigger() throws Exception { TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new ResultSortComparator()); testHarness.close(); - if (windowBufferFactory instanceof PreAggregatingHeapWindowBuffer.Factory) { - Assert.assertEquals("Close was not called.", 2, closeCalled.get()); - } else { - Assert.assertEquals("Close was not called.", 1, closeCalled.get()); - } } @Test @@ -316,7 +302,7 @@ public void testCountTrigger() throws Exception { GlobalWindows.create(), new GlobalWindow.Serializer(), windowBufferFactory, - new ReduceAllWindowFunction>(new SumReducer()), + new ReduceIterableAllWindowFunction>(new SumReducer()), PurgingTrigger.of(CountTrigger.of(WINDOW_SIZE))); operator.setInputType(TypeInfoParser.>parse( @@ -355,19 +341,23 @@ public void testCountTrigger() throws Exception { TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new ResultSortComparator()); testHarness.close(); - if (windowBufferFactory instanceof PreAggregatingHeapWindowBuffer.Factory) { - Assert.assertEquals("Close was not called.", 2, closeCalled.get()); - } else { - Assert.assertEquals("Close was not called.", 1, closeCalled.get()); - } - } // ------------------------------------------------------------------------ // UDFs // ------------------------------------------------------------------------ - public static class SumReducer extends RichReduceFunction> { + public static class SumReducer implements ReduceFunction> { + private static final long serialVersionUID = 1L; + + @Override + public Tuple2 reduce(Tuple2 value1, + Tuple2 value2) throws Exception { + return new Tuple2<>(value2.f0, value1.f1 + value2.f1); + } + } + + public static class RichSumReducer extends RichReduceFunction> { private static final long serialVersionUID = 1L; private boolean openCalled = false; @@ -400,7 +390,7 @@ public Tuple2 reduce(Tuple2 value1, @Parameterized.Parameters(name = "WindowBuffer = {0}") @SuppressWarnings("unchecked,rawtypes") public static Collection windowBuffers(){ - return Arrays.asList(new WindowBufferFactory[]{new PreAggregatingHeapWindowBuffer.Factory(new SumReducer())}, + return Arrays.asList(new WindowBufferFactory[]{new PreAggregatingHeapWindowBuffer.Factory(new RichSumReducer())}, new WindowBufferFactory[]{new HeapWindowBuffer.Factory()} ); } diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/TimeWindowTranslationTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/TimeWindowTranslationTest.java index 76c6f20c089e5..b99232ae3ce58 100644 --- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/TimeWindowTranslationTest.java +++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/TimeWindowTranslationTest.java @@ -17,7 +17,7 @@ */ package org.apache.flink.streaming.runtime.operators.windowing; -import org.apache.flink.api.common.functions.RichReduceFunction; +import org.apache.flink.api.common.functions.ReduceFunction; import org.apache.flink.api.java.tuple.Tuple; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.streaming.api.datastream.DataStream; @@ -68,7 +68,7 @@ public void testFastTimeWindows() throws Exception { DataStream> window2 = source .keyBy(0) .timeWindow(Time.of(1000, TimeUnit.MILLISECONDS)) - .apply(new WindowFunction, Tuple2, Tuple, TimeWindow>() { + .apply(new WindowFunction>, Tuple2, Tuple, TimeWindow>() { private static final long serialVersionUID = 1L; @Override @@ -111,7 +111,7 @@ public void testNonParallelFastTimeWindows() throws Exception { DataStream> window2 = source .timeWindowAll(Time.of(1000, TimeUnit.MILLISECONDS)) - .apply(new AllWindowFunction, Tuple2, TimeWindow>() { + .apply(new AllWindowFunction>, Tuple2, TimeWindow>() { private static final long serialVersionUID = 1L; @Override @@ -132,7 +132,7 @@ public void apply( // UDFs // ------------------------------------------------------------------------ - public static class DummyReducer extends RichReduceFunction> { + public static class DummyReducer implements ReduceFunction> { private static final long serialVersionUID = 1L; @Override diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/WindowOperatorTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/WindowOperatorTest.java index b94e53018dfd9..9d4a41a0b0da3 100644 --- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/WindowOperatorTest.java +++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/WindowOperatorTest.java @@ -18,20 +18,22 @@ package org.apache.flink.streaming.runtime.operators.windowing; import org.apache.flink.api.common.ExecutionConfig; -import org.apache.flink.api.common.functions.RichReduceFunction; +import org.apache.flink.api.common.functions.ReduceFunction; +import org.apache.flink.api.common.state.ListStateDescriptor; +import org.apache.flink.api.common.state.ReducingStateDescriptor; import org.apache.flink.api.common.typeinfo.BasicTypeInfo; +import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.functions.KeySelector; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.api.java.typeutils.TypeInfoParser; import org.apache.flink.configuration.Configuration; +import org.apache.flink.streaming.api.functions.windowing.RichWindowFunction; import org.apache.flink.streaming.api.watermark.Watermark; import org.apache.flink.streaming.api.windowing.assigners.GlobalWindows; import org.apache.flink.streaming.api.windowing.assigners.SlidingTimeWindows; import org.apache.flink.streaming.api.windowing.assigners.TumblingTimeWindows; import org.apache.flink.streaming.api.windowing.time.Time; -import org.apache.flink.streaming.runtime.operators.windowing.buffers.HeapWindowBuffer; -import org.apache.flink.streaming.runtime.operators.windowing.buffers.PreAggregatingHeapWindowBuffer; -import org.apache.flink.streaming.runtime.operators.windowing.buffers.WindowBufferFactory; +import org.apache.flink.streaming.api.windowing.windows.Window; import org.apache.flink.streaming.api.functions.windowing.ReduceWindowFunction; import org.apache.flink.streaming.api.windowing.triggers.ContinuousEventTimeTrigger; import org.apache.flink.streaming.api.windowing.triggers.CountTrigger; @@ -42,57 +44,25 @@ import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; import org.apache.flink.streaming.util.OneInputStreamOperatorTestHarness; import org.apache.flink.streaming.util.TestHarnessUtil; +import org.apache.flink.util.Collector; import org.junit.Assert; import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; -import java.util.Arrays; -import java.util.Collection; import java.util.Comparator; import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; -@RunWith(Parameterized.class) public class WindowOperatorTest { - @SuppressWarnings("unchecked,rawtypes") - private WindowBufferFactory windowBufferFactory; - - public WindowOperatorTest(WindowBufferFactory windowBufferFactory) { - this.windowBufferFactory = windowBufferFactory; - } - // For counting if close() is called the correct number of times on the SumReducer private static AtomicInteger closeCalled = new AtomicInteger(0); - @Test - @SuppressWarnings("unchecked") - public void testSlidingEventTimeWindows() throws Exception { - closeCalled.set(0); - - final int WINDOW_SIZE = 3; - final int WINDOW_SLIDE = 1; - - WindowOperator, Tuple2, TimeWindow> operator = new WindowOperator<>( - SlidingTimeWindows.of(Time.of(WINDOW_SIZE, TimeUnit.SECONDS), Time.of(WINDOW_SLIDE, TimeUnit.SECONDS)), - new TimeWindow.Serializer(), - new TupleKeySelector(), - BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()), - windowBufferFactory, - new ReduceWindowFunction>(new SumReducer()), - EventTimeTrigger.create()); - - operator.setInputType(TypeInfoParser.>parse("Tuple2"), new ExecutionConfig()); - - OneInputStreamOperatorTestHarness, Tuple2> testHarness = - new OneInputStreamOperatorTestHarness<>(operator); + private void testSlidingEventTimeWindows(OneInputStreamOperatorTestHarness, Tuple2> testHarness) throws Exception { long initialTime = 0L; - ConcurrentLinkedQueue expectedOutput = new ConcurrentLinkedQueue<>(); - testHarness.open(); + ConcurrentLinkedQueue expectedOutput = new ConcurrentLinkedQueue<>(); // add elements out-of-order testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 3999)); @@ -148,37 +118,84 @@ public void testSlidingEventTimeWindows() throws Exception { expectedOutput.add(new Watermark(7999)); TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new ResultSortComparator()); - - testHarness.close(); - if (windowBufferFactory instanceof PreAggregatingHeapWindowBuffer.Factory) { - Assert.assertEquals("Close was not called.", 2, closeCalled.get()); - } else { - Assert.assertEquals("Close was not called.", 1, closeCalled.get()); - } } @Test @SuppressWarnings("unchecked") - public void testTumblingEventTimeWindows() throws Exception { + public void testSlidingEventTimeWindowsReduce() throws Exception { closeCalled.set(0); final int WINDOW_SIZE = 3; + final int WINDOW_SLIDE = 1; - WindowOperator, Tuple2, TimeWindow> operator = new WindowOperator<>( - TumblingTimeWindows.of(Time.of(WINDOW_SIZE, TimeUnit.SECONDS)), + TypeInformation> inputType = TypeInfoParser.parse("Tuple2"); + + ReducingStateDescriptor> stateDesc = new ReducingStateDescriptor<>("window-contents", + new SumReducer(), + inputType.createSerializer(new ExecutionConfig())); + + WindowOperator, Tuple2, Tuple2, TimeWindow> operator = new WindowOperator<>( + SlidingTimeWindows.of(Time.of(WINDOW_SIZE, TimeUnit.SECONDS), Time.of(WINDOW_SLIDE, TimeUnit.SECONDS)), new TimeWindow.Serializer(), new TupleKeySelector(), BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()), - windowBufferFactory, - new ReduceWindowFunction>(new SumReducer()), + stateDesc, + new ReduceWindowFunction>(), EventTimeTrigger.create()); - operator.setInputType(TypeInfoParser.>parse("Tuple2"), new ExecutionConfig()); - + operator.setInputType(inputType, new ExecutionConfig()); OneInputStreamOperatorTestHarness, Tuple2> testHarness = new OneInputStreamOperatorTestHarness<>(operator); + testHarness.configureForKeyedStream(new TupleKeySelector(), BasicTypeInfo.STRING_TYPE_INFO); + + testHarness.open(); + + testSlidingEventTimeWindows(testHarness); + + testHarness.close(); + } + + @Test + @SuppressWarnings("unchecked") + public void testSlidingEventTimeWindowsApply() throws Exception { + closeCalled.set(0); + + final int WINDOW_SIZE = 3; + final int WINDOW_SLIDE = 1; + + TypeInformation> inputType = TypeInfoParser.parse("Tuple2"); + + ListStateDescriptor> stateDesc = new ListStateDescriptor<>("window-contents", + inputType.createSerializer(new ExecutionConfig())); + + WindowOperator, Iterable>, Tuple2, TimeWindow> operator = new WindowOperator<>( + SlidingTimeWindows.of(Time.of(WINDOW_SIZE, TimeUnit.SECONDS), Time.of(WINDOW_SLIDE, TimeUnit.SECONDS)), + new TimeWindow.Serializer(), + new TupleKeySelector(), + BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()), + stateDesc, + new RichSumReducer(), + EventTimeTrigger.create()); + + operator.setInputType(inputType, new ExecutionConfig()); + + OneInputStreamOperatorTestHarness, Tuple2> testHarness = + new OneInputStreamOperatorTestHarness<>(operator); + + testHarness.configureForKeyedStream(new TupleKeySelector(), BasicTypeInfo.STRING_TYPE_INFO); + + testHarness.open(); + + testSlidingEventTimeWindows(testHarness); + + testHarness.close(); + + Assert.assertEquals("Close was not called.", 1, closeCalled.get()); + } + + private void testTumblingEventTimeWindows(OneInputStreamOperatorTestHarness, Tuple2> testHarness) throws Exception { long initialTime = 0L; ConcurrentLinkedQueue expectedOutput = new ConcurrentLinkedQueue<>(); @@ -233,13 +250,79 @@ public void testTumblingEventTimeWindows() throws Exception { expectedOutput.add(new Watermark(7999)); TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new ResultSortComparator()); + } + + @Test + @SuppressWarnings("unchecked") + public void testTumblingEventTimeWindowsReduce() throws Exception { + closeCalled.set(0); + + final int WINDOW_SIZE = 3; + + TypeInformation> inputType = TypeInfoParser.parse("Tuple2"); + + ReducingStateDescriptor> stateDesc = new ReducingStateDescriptor<>("window-contents", + new SumReducer(), + inputType.createSerializer(new ExecutionConfig())); + + WindowOperator, Tuple2, Tuple2, TimeWindow> operator = new WindowOperator<>( + TumblingTimeWindows.of(Time.of(WINDOW_SIZE, TimeUnit.SECONDS)), + new TimeWindow.Serializer(), + new TupleKeySelector(), + BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()), + stateDesc, + new ReduceWindowFunction>(), + EventTimeTrigger.create()); + + operator.setInputType(TypeInfoParser.>parse("Tuple2"), new ExecutionConfig()); + + OneInputStreamOperatorTestHarness, Tuple2> testHarness = + new OneInputStreamOperatorTestHarness<>(operator); + + testHarness.configureForKeyedStream(new TupleKeySelector(), BasicTypeInfo.STRING_TYPE_INFO); + + testHarness.open(); + + testTumblingEventTimeWindows(testHarness); testHarness.close(); - if (windowBufferFactory instanceof PreAggregatingHeapWindowBuffer.Factory) { - Assert.assertEquals("Close was not called.", 2, closeCalled.get()); - } else { - Assert.assertEquals("Close was not called.", 1, closeCalled.get()); - } + } + + @Test + @SuppressWarnings("unchecked") + public void testTumblingEventTimeWindowsApply() throws Exception { + closeCalled.set(0); + + final int WINDOW_SIZE = 3; + + TypeInformation> inputType = TypeInfoParser.parse("Tuple2"); + + ListStateDescriptor> stateDesc = new ListStateDescriptor<>("window-contents", + inputType.createSerializer(new ExecutionConfig())); + + WindowOperator, Iterable>, Tuple2, TimeWindow> operator = new WindowOperator<>( + TumblingTimeWindows.of(Time.of(WINDOW_SIZE, TimeUnit.SECONDS)), + new TimeWindow.Serializer(), + new TupleKeySelector(), + BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()), + stateDesc, + new RichSumReducer(), + EventTimeTrigger.create()); + + operator.setInputType(TypeInfoParser.>parse("Tuple2"), new ExecutionConfig()); + + OneInputStreamOperatorTestHarness, Tuple2> testHarness = + new OneInputStreamOperatorTestHarness<>(operator); + + testHarness.configureForKeyedStream(new TupleKeySelector(), BasicTypeInfo.STRING_TYPE_INFO); + + testHarness.open(); + + testTumblingEventTimeWindows(testHarness); + + testHarness.close(); + + Assert.assertEquals("Close was not called.", 1, closeCalled.get()); } @Test @@ -249,13 +332,19 @@ public void testContinuousWatermarkTrigger() throws Exception { final int WINDOW_SIZE = 3; - WindowOperator, Tuple2, GlobalWindow> operator = new WindowOperator<>( + TypeInformation> inputType = TypeInfoParser.parse("Tuple2"); + + ReducingStateDescriptor> stateDesc = new ReducingStateDescriptor<>("window-contents", + new SumReducer(), + inputType.createSerializer(new ExecutionConfig())); + + WindowOperator, Tuple2, Tuple2, GlobalWindow> operator = new WindowOperator<>( GlobalWindows.create(), new GlobalWindow.Serializer(), new TupleKeySelector(), BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()), - windowBufferFactory, - new ReduceWindowFunction>(new SumReducer()), + stateDesc, + new ReduceWindowFunction>(), ContinuousEventTimeTrigger.of(Time.of(WINDOW_SIZE, TimeUnit.SECONDS))); operator.setInputType(TypeInfoParser.>parse("Tuple2"), new ExecutionConfig()); @@ -263,6 +352,8 @@ public void testContinuousWatermarkTrigger() throws Exception { OneInputStreamOperatorTestHarness, Tuple2> testHarness = new OneInputStreamOperatorTestHarness<>(operator); + testHarness.configureForKeyedStream(new TupleKeySelector(), BasicTypeInfo.STRING_TYPE_INFO); + long initialTime = 0L; ConcurrentLinkedQueue expectedOutput = new ConcurrentLinkedQueue<>(); @@ -322,11 +413,6 @@ public void testContinuousWatermarkTrigger() throws Exception { TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new ResultSortComparator()); testHarness.close(); - if (windowBufferFactory instanceof PreAggregatingHeapWindowBuffer.Factory) { - Assert.assertEquals("Close was not called.", 2, closeCalled.get()); - } else { - Assert.assertEquals("Close was not called.", 1, closeCalled.get()); - } } @Test @@ -336,13 +422,19 @@ public void testCountTrigger() throws Exception { final int WINDOW_SIZE = 4; - WindowOperator, Tuple2, GlobalWindow> operator = new WindowOperator<>( + TypeInformation> inputType = TypeInfoParser.parse("Tuple2"); + + ReducingStateDescriptor> stateDesc = new ReducingStateDescriptor<>("window-contents", + new SumReducer(), + inputType.createSerializer(new ExecutionConfig())); + + WindowOperator, Tuple2, Tuple2, GlobalWindow> operator = new WindowOperator<>( GlobalWindows.create(), new GlobalWindow.Serializer(), new TupleKeySelector(), BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()), - windowBufferFactory, - new ReduceWindowFunction>(new SumReducer()), + stateDesc, + new ReduceWindowFunction>(), PurgingTrigger.of(CountTrigger.of(WINDOW_SIZE))); operator.setInputType(TypeInfoParser.>parse( @@ -351,6 +443,8 @@ public void testCountTrigger() throws Exception { OneInputStreamOperatorTestHarness, Tuple2> testHarness = new OneInputStreamOperatorTestHarness<>(operator); + testHarness.configureForKeyedStream(new TupleKeySelector(), BasicTypeInfo.STRING_TYPE_INFO); + long initialTime = 0L; ConcurrentLinkedQueue expectedOutput = new ConcurrentLinkedQueue<>(); @@ -387,19 +481,23 @@ public void testCountTrigger() throws Exception { TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new ResultSortComparator()); testHarness.close(); - if (windowBufferFactory instanceof PreAggregatingHeapWindowBuffer.Factory) { - Assert.assertEquals("Close was not called.", 2, closeCalled.get()); - } else { - Assert.assertEquals("Close was not called.", 1, closeCalled.get()); - } - } // ------------------------------------------------------------------------ // UDFs // ------------------------------------------------------------------------ - public static class SumReducer extends RichReduceFunction> { + public static class SumReducer implements ReduceFunction> { + private static final long serialVersionUID = 1L; + @Override + public Tuple2 reduce(Tuple2 value1, + Tuple2 value2) throws Exception { + return new Tuple2<>(value2.f0, value1.f1 + value2.f1); + } + } + + + public static class RichSumReducer extends RichWindowFunction>, Tuple2, String, W> { private static final long serialVersionUID = 1L; private boolean openCalled = false; @@ -417,24 +515,23 @@ public void close() throws Exception { } @Override - public Tuple2 reduce(Tuple2 value1, - Tuple2 value2) throws Exception { + public void apply(String key, + W window, + Iterable> input, + Collector> out) throws Exception { + if (!openCalled) { Assert.fail("Open was not called"); } - return new Tuple2<>(value2.f0, value1.f1 + value2.f1); + int sum = 0; + + for (Tuple2 t: input) { + sum += t.f1; + } + out.collect(new Tuple2<>(key, sum)); + } - } - // ------------------------------------------------------------------------ - // Parametrization for testing different window buffers - // ------------------------------------------------------------------------ - @Parameterized.Parameters(name = "WindowBuffer = {0}") - @SuppressWarnings("unchecked,rawtypes") - public static Collection windowBuffers(){ - return Arrays.asList(new WindowBufferFactory[]{new PreAggregatingHeapWindowBuffer.Factory(new SumReducer())}, - new WindowBufferFactory[]{new HeapWindowBuffer.Factory()} - ); } @SuppressWarnings("unchecked") diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/WindowTranslationTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/WindowTranslationTest.java index 13766a19e75d6..1e6e47520a76e 100644 --- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/WindowTranslationTest.java +++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/WindowTranslationTest.java @@ -17,7 +17,10 @@ */ package org.apache.flink.streaming.runtime.operators.windowing; +import org.apache.flink.api.common.functions.ReduceFunction; import org.apache.flink.api.common.functions.RichReduceFunction; +import org.apache.flink.api.common.state.ListStateDescriptor; +import org.apache.flink.api.common.state.ReducingStateDescriptor; import org.apache.flink.api.java.tuple.Tuple; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.streaming.api.TimeCharacteristic; @@ -35,8 +38,6 @@ import org.apache.flink.streaming.api.windowing.triggers.CountTrigger; import org.apache.flink.streaming.api.windowing.triggers.EventTimeTrigger; import org.apache.flink.streaming.api.windowing.windows.TimeWindow; -import org.apache.flink.streaming.runtime.operators.windowing.buffers.HeapWindowBuffer; -import org.apache.flink.streaming.runtime.operators.windowing.buffers.PreAggregatingHeapWindowBuffer; import org.apache.flink.streaming.util.StreamingMultipleProgramsTestBase; import org.apache.flink.util.Collector; import org.junit.Assert; @@ -51,6 +52,29 @@ */ public class WindowTranslationTest extends StreamingMultipleProgramsTestBase { + /** + * .reduce() does not support RichReduceFunction, since the reduce function is used internally + * in a {@code ReducingState}. + */ + @Test(expected = UnsupportedOperationException.class) + public void testReduceFailWithRichReducer() throws Exception { + StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); + + DataStream> source = env.fromElements(Tuple2.of("hello", 1), Tuple2.of("hello", 2)); + env.setStreamTimeCharacteristic(TimeCharacteristic.ProcessingTime); + + DataStream> window1 = source + .keyBy(0) + .window(SlidingTimeWindows.of(Time.of(1, TimeUnit.SECONDS), Time.of(100, TimeUnit.MILLISECONDS))) + .reduce(new RichReduceFunction>() { + @Override + public Tuple2 reduce(Tuple2 value1, + Tuple2 value2) throws Exception { + return null; + } + }); + } + /** * These tests ensure that the fast aligned time windows operator is used if the * conditions are right. @@ -76,7 +100,7 @@ public void testFastTimeWindows() throws Exception { DataStream> window2 = source .keyBy(0) .window(SlidingTimeWindows.of(Time.of(1, TimeUnit.SECONDS), Time.of(100, TimeUnit.MILLISECONDS))) - .apply(new WindowFunction, Tuple2, Tuple, TimeWindow>() { + .apply(new WindowFunction>, Tuple2, Tuple, TimeWindow>() { private static final long serialVersionUID = 1L; @Override @@ -118,12 +142,12 @@ public void testEventTime() throws Exception { Assert.assertFalse(winOperator1.isSetProcessingTime()); Assert.assertTrue(winOperator1.getTrigger() instanceof EventTimeTrigger); Assert.assertTrue(winOperator1.getWindowAssigner() instanceof SlidingTimeWindows); - Assert.assertTrue(winOperator1.getWindowBufferFactory() instanceof PreAggregatingHeapWindowBuffer.Factory); + Assert.assertTrue(winOperator1.getStateDescriptor() instanceof ReducingStateDescriptor); DataStream> window2 = source .keyBy(0) .window(TumblingTimeWindows.of(Time.of(1, TimeUnit.SECONDS))) - .apply(new WindowFunction, Tuple2, Tuple, TimeWindow>() { + .apply(new WindowFunction>, Tuple2, Tuple, TimeWindow>() { private static final long serialVersionUID = 1L; @Override @@ -142,7 +166,7 @@ public void apply(Tuple tuple, Assert.assertFalse(winOperator2.isSetProcessingTime()); Assert.assertTrue(winOperator2.getTrigger() instanceof EventTimeTrigger); Assert.assertTrue(winOperator2.getWindowAssigner() instanceof TumblingTimeWindows); - Assert.assertTrue(winOperator2.getWindowBufferFactory() instanceof HeapWindowBuffer.Factory); + Assert.assertTrue(winOperator2.getStateDescriptor() instanceof ListStateDescriptor); } @Test @@ -168,13 +192,13 @@ public void testNonEvicting() throws Exception { Assert.assertTrue(winOperator1.isSetProcessingTime()); Assert.assertTrue(winOperator1.getTrigger() instanceof CountTrigger); Assert.assertTrue(winOperator1.getWindowAssigner() instanceof SlidingTimeWindows); - Assert.assertTrue(winOperator1.getWindowBufferFactory() instanceof PreAggregatingHeapWindowBuffer.Factory); + Assert.assertTrue(winOperator1.getStateDescriptor() instanceof ReducingStateDescriptor); DataStream> window2 = source .keyBy(0) .window(TumblingTimeWindows.of(Time.of(1, TimeUnit.SECONDS))) .trigger(CountTrigger.of(100)) - .apply(new WindowFunction, Tuple2, Tuple, TimeWindow>() { + .apply(new WindowFunction>, Tuple2, Tuple, TimeWindow>() { private static final long serialVersionUID = 1L; @Override @@ -193,7 +217,7 @@ public void apply(Tuple tuple, Assert.assertTrue(winOperator2.isSetProcessingTime()); Assert.assertTrue(winOperator2.getTrigger() instanceof CountTrigger); Assert.assertTrue(winOperator2.getWindowAssigner() instanceof TumblingTimeWindows); - Assert.assertTrue(winOperator2.getWindowBufferFactory() instanceof HeapWindowBuffer.Factory); + Assert.assertTrue(winOperator2.getStateDescriptor() instanceof ListStateDescriptor); } @Test @@ -220,14 +244,14 @@ public void testEvicting() throws Exception { Assert.assertTrue(winOperator1.getTrigger() instanceof EventTimeTrigger); Assert.assertTrue(winOperator1.getWindowAssigner() instanceof SlidingTimeWindows); Assert.assertTrue(winOperator1.getEvictor() instanceof CountEvictor); - Assert.assertTrue(winOperator1.getWindowBufferFactory() instanceof HeapWindowBuffer.Factory); + Assert.assertTrue(winOperator1.getStateDescriptor() instanceof ListStateDescriptor); DataStream> window2 = source .keyBy(0) .window(TumblingTimeWindows.of(Time.of(1, TimeUnit.SECONDS))) .trigger(CountTrigger.of(100)) .evictor(TimeEvictor.of(Time.of(100, TimeUnit.MILLISECONDS))) - .apply(new WindowFunction, Tuple2, Tuple, TimeWindow>() { + .apply(new WindowFunction>, Tuple2, Tuple, TimeWindow>() { private static final long serialVersionUID = 1L; @Override @@ -247,14 +271,14 @@ public void apply(Tuple tuple, Assert.assertTrue(winOperator2.getTrigger() instanceof CountTrigger); Assert.assertTrue(winOperator2.getWindowAssigner() instanceof TumblingTimeWindows); Assert.assertTrue(winOperator2.getEvictor() instanceof TimeEvictor); - Assert.assertTrue(winOperator2.getWindowBufferFactory() instanceof HeapWindowBuffer.Factory); + Assert.assertTrue(winOperator2.getStateDescriptor() instanceof ListStateDescriptor); } // ------------------------------------------------------------------------ // UDFs // ------------------------------------------------------------------------ - public static class DummyReducer extends RichReduceFunction> { + public static class DummyReducer implements ReduceFunction> { private static final long serialVersionUID = 1L; @Override diff --git a/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/AllWindowedStream.scala b/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/AllWindowedStream.scala index 0357144340c5f..90e63c4b93402 100644 --- a/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/AllWindowedStream.scala +++ b/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/AllWindowedStream.scala @@ -176,8 +176,15 @@ class AllWindowedStream[T, W <: Window](javaStream: JavaAllWStream[T, W]) { * @param function The window function. * @return The data stream that is the result of applying the window function to the window. */ - def apply[R: TypeInformation: ClassTag](function: AllWindowFunction[T, R, W]): DataStream[R] = { - javaStream.apply(clean(function), implicitly[TypeInformation[R]]) + def apply[R: TypeInformation: ClassTag]( + function: AllWindowFunction[Iterable[T], R, W]): DataStream[R] = { + val cleanedFunction = clean(function) + val javaFunction = new AllWindowFunction[java.lang.Iterable[T], R, W] { + def apply(window: W, elements: java.lang.Iterable[T], out: Collector[R]): Unit = { + cleanedFunction(window, elements.asScala, out) + } + } + javaStream.apply(javaFunction, implicitly[TypeInformation[R]]) } /** @@ -194,7 +201,7 @@ class AllWindowedStream[T, W <: Window](javaStream: JavaAllWStream[T, W]) { def apply[R: TypeInformation: ClassTag]( function: (W, Iterable[T], Collector[R]) => Unit): DataStream[R] = { val cleanedFunction = clean(function) - val applyFunction = new AllWindowFunction[T, R, W] { + val applyFunction = new AllWindowFunction[java.lang.Iterable[T], R, W] { def apply(window: W, elements: java.lang.Iterable[T], out: Collector[R]): Unit = { cleanedFunction(window, elements.asScala, out) } @@ -232,7 +239,7 @@ class AllWindowedStream[T, W <: Window](javaStream: JavaAllWStream[T, W]) { */ def apply[R: TypeInformation: ClassTag]( preAggregator: (T, T) => T, - function: (W, Iterable[T], Collector[R]) => Unit): DataStream[R] = { + function: (W, T, Collector[R]) => Unit): DataStream[R] = { if (function == null) { throw new NullPointerException("Reduce function must not be null.") } @@ -247,8 +254,8 @@ class AllWindowedStream[T, W <: Window](javaStream: JavaAllWStream[T, W]) { val cleanApply = clean(function) val applyFunction = new AllWindowFunction[T, R, W] { - def apply(window: W, elements: java.lang.Iterable[T], out: Collector[R]): Unit = { - cleanApply(window, elements.asScala, out) + def apply(window: W, input: T, out: Collector[R]): Unit = { + cleanApply(window, input, out) } } javaStream.apply(reducer, applyFunction, implicitly[TypeInformation[R]]) diff --git a/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/WindowedStream.scala b/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/WindowedStream.scala index 93b91ffbd23b9..8a49f4063d3b0 100644 --- a/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/WindowedStream.scala +++ b/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/WindowedStream.scala @@ -179,8 +179,15 @@ class WindowedStream[T, K, W <: Window](javaStream: JavaWStream[T, K, W]) { * @param function The window function. * @return The data stream that is the result of applying the window function to the window. */ - def apply[R: TypeInformation: ClassTag](function: WindowFunction[T, R, K, W]): DataStream[R] = { - javaStream.apply(clean(function), implicitly[TypeInformation[R]]) + def apply[R: TypeInformation: ClassTag]( + function: WindowFunction[Iterable[T], R, K, W]): DataStream[R] = { + val cleanFunction = clean(function) + val javaFunction = new WindowFunction[java.lang.Iterable[T], R, K, W] { + def apply(key: K, window: W, input: java.lang.Iterable[T], out: Collector[R]) = { + cleanFunction.apply(key, window, input.asScala, out) + } + } + javaStream.apply(javaFunction, implicitly[TypeInformation[R]]) } /** @@ -201,7 +208,7 @@ class WindowedStream[T, K, W <: Window](javaStream: JavaWStream[T, K, W]) { } val cleanedFunction = clean(function) - val applyFunction = new WindowFunction[T, R, K, W] { + val applyFunction = new WindowFunction[java.lang.Iterable[T], R, K, W] { def apply(key: K, window: W, elements: java.lang.Iterable[T], out: Collector[R]): Unit = { cleanedFunction(key, window, elements.asScala, out) } @@ -239,7 +246,7 @@ class WindowedStream[T, K, W <: Window](javaStream: JavaWStream[T, K, W]) { */ def apply[R: TypeInformation: ClassTag]( preAggregator: (T, T) => T, - function: (K, W, Iterable[T], Collector[R]) => Unit): DataStream[R] = { + function: (K, W, T, Collector[R]) => Unit): DataStream[R] = { if (function == null) { throw new NullPointerException("Reduce function must not be null.") } @@ -254,8 +261,8 @@ class WindowedStream[T, K, W <: Window](javaStream: JavaWStream[T, K, W]) { val cleanApply = clean(function) val applyFunction = new WindowFunction[T, R, K, W] { - def apply(key: K, window: W, elements: java.lang.Iterable[T], out: Collector[R]): Unit = { - cleanApply(key, window, elements.asScala, out) + def apply(key: K, window: W, input: T, out: Collector[R]): Unit = { + cleanApply(key, window, input, out) } } javaStream.apply(reducer, applyFunction, implicitly[TypeInformation[R]]) diff --git a/flink-streaming-scala/src/test/scala/org/apache/flink/streaming/api/scala/AllWindowTranslationTest.scala b/flink-streaming-scala/src/test/scala/org/apache/flink/streaming/api/scala/AllWindowTranslationTest.scala index 7da7bc3d591d6..217da25518b84 100644 --- a/flink-streaming-scala/src/test/scala/org/apache/flink/streaming/api/scala/AllWindowTranslationTest.scala +++ b/flink-streaming-scala/src/test/scala/org/apache/flink/streaming/api/scala/AllWindowTranslationTest.scala @@ -21,7 +21,8 @@ package org.apache.flink.streaming.api.scala import java.util.concurrent.TimeUnit -import org.apache.flink.api.common.functions.RichReduceFunction +import org.apache.flink.api.common.functions.ReduceFunction +import org.apache.flink.api.common.state.ReducingStateDescriptor import org.apache.flink.api.java.tuple.Tuple import org.apache.flink.streaming.api.TimeCharacteristic import org.apache.flink.streaming.api.functions.windowing.{WindowFunction, AllWindowFunction} @@ -75,12 +76,12 @@ class AllWindowTranslationTest extends StreamingMultipleProgramsTestBase { .windowAll(SlidingTimeWindows.of( Time.of(1, TimeUnit.SECONDS), Time.of(100, TimeUnit.MILLISECONDS))) - .apply(new AllWindowFunction[(String, Int), (String, Int), TimeWindow]() { - def apply( - window: TimeWindow, - values: java.lang.Iterable[(String, Int)], - out: Collector[(String, Int)]) { } - }) + .apply(new AllWindowFunction[Iterable[(String, Int)], (String, Int), TimeWindow]() { + def apply( + window: TimeWindow, + values: Iterable[(String, Int)], + out: Collector[(String, Int)]) { } + }) val transform2 = window2.getJavaStream.getTransformation .asInstanceOf[OneInputTransformation[(String, Int), (String, Int)]] @@ -121,10 +122,10 @@ class AllWindowTranslationTest extends StreamingMultipleProgramsTestBase { val window2 = source .windowAll(TumblingTimeWindows.of(Time.of(1, TimeUnit.SECONDS))) .trigger(CountTrigger.of(100)) - .apply(new AllWindowFunction[(String, Int), (String, Int), TimeWindow]() { + .apply(new AllWindowFunction[Iterable[(String, Int)], (String, Int), TimeWindow]() { def apply( window: TimeWindow, - values: java.lang.Iterable[(String, Int)], + values: Iterable[(String, Int)], out: Collector[(String, Int)]) { } }) @@ -172,10 +173,10 @@ class AllWindowTranslationTest extends StreamingMultipleProgramsTestBase { .windowAll(TumblingTimeWindows.of(Time.of(1, TimeUnit.SECONDS))) .trigger(CountTrigger.of(100)) .evictor(CountEvictor.of(1000)) - .apply(new AllWindowFunction[(String, Int), (String, Int), TimeWindow]() { + .apply(new AllWindowFunction[Iterable[(String, Int)], (String, Int), TimeWindow]() { def apply( window: TimeWindow, - values: java.lang.Iterable[(String, Int)], + values: Iterable[(String, Int)], out: Collector[(String, Int)]) { } }) @@ -210,7 +211,7 @@ class AllWindowTranslationTest extends StreamingMultipleProgramsTestBase { def apply( tuple: Tuple, window: TimeWindow, - values: java.lang.Iterable[(String, Int)], + values: (String, Int), out: Collector[(String, Int)]) { } }) @@ -219,12 +220,12 @@ class AllWindowTranslationTest extends StreamingMultipleProgramsTestBase { val operator1 = transform1.getOperator - assertTrue(operator1.isInstanceOf[WindowOperator[_, _, _, _]]) - val winOperator1 = operator1.asInstanceOf[WindowOperator[_, _, _, _]] + assertTrue(operator1.isInstanceOf[WindowOperator[_, _, _, _, _]]) + val winOperator1 = operator1.asInstanceOf[WindowOperator[_, _, _, _, _]] assertTrue(winOperator1.getTrigger.isInstanceOf[CountTrigger[_]]) assertTrue(winOperator1.getWindowAssigner.isInstanceOf[SlidingTimeWindows]) assertTrue( - winOperator1.getWindowBufferFactory.isInstanceOf[PreAggregatingHeapWindowBuffer.Factory[_]]) + winOperator1.getStateDescriptor.isInstanceOf[ReducingStateDescriptor[_]]) val window2 = source @@ -235,7 +236,7 @@ class AllWindowTranslationTest extends StreamingMultipleProgramsTestBase { def apply( tuple: Tuple, window: TimeWindow, - values: java.lang.Iterable[(String, Int)], + values: (String, Int), out: Collector[(String, Int)]) { } }) @@ -244,12 +245,12 @@ class AllWindowTranslationTest extends StreamingMultipleProgramsTestBase { val operator2 = transform2.getOperator - assertTrue(operator2.isInstanceOf[WindowOperator[_, _, _, _]]) - val winOperator2 = operator2.asInstanceOf[WindowOperator[_, _, _, _]] + assertTrue(operator2.isInstanceOf[WindowOperator[_, _, _, _, _]]) + val winOperator2 = operator2.asInstanceOf[WindowOperator[_, _, _, _, _]] assertTrue(winOperator2.getTrigger.isInstanceOf[CountTrigger[_]]) assertTrue(winOperator2.getWindowAssigner.isInstanceOf[TumblingTimeWindows]) assertTrue( - winOperator2.getWindowBufferFactory.isInstanceOf[PreAggregatingHeapWindowBuffer.Factory[_]]) + winOperator2.getStateDescriptor.isInstanceOf[ReducingStateDescriptor[_]]) } } @@ -258,7 +259,7 @@ class AllWindowTranslationTest extends StreamingMultipleProgramsTestBase { // UDFs // ------------------------------------------------------------------------ -class DummyReducer extends RichReduceFunction[(String, Int)] { +class DummyReducer extends ReduceFunction[(String, Int)] { def reduce(value1: (String, Int), value2: (String, Int)): (String, Int) = { value1 } diff --git a/flink-streaming-scala/src/test/scala/org/apache/flink/streaming/api/scala/WindowTranslationTest.scala b/flink-streaming-scala/src/test/scala/org/apache/flink/streaming/api/scala/WindowTranslationTest.scala index 46981ab873a57..e43dc6ed50dd2 100644 --- a/flink-streaming-scala/src/test/scala/org/apache/flink/streaming/api/scala/WindowTranslationTest.scala +++ b/flink-streaming-scala/src/test/scala/org/apache/flink/streaming/api/scala/WindowTranslationTest.scala @@ -20,6 +20,7 @@ package org.apache.flink.streaming.api.scala import java.util.concurrent.TimeUnit +import org.apache.flink.api.common.state.{ListStateDescriptor, ReducingStateDescriptor} import org.apache.flink.api.java.tuple.Tuple import org.apache.flink.streaming.api.functions.windowing.WindowFunction import org.apache.flink.streaming.api.transformations.OneInputTransformation @@ -28,7 +29,6 @@ import org.apache.flink.streaming.api.windowing.evictors.{CountEvictor, TimeEvic import org.apache.flink.streaming.api.windowing.time.Time import org.apache.flink.streaming.api.windowing.triggers.{ProcessingTimeTrigger, CountTrigger} import org.apache.flink.streaming.api.windowing.windows.TimeWindow -import org.apache.flink.streaming.runtime.operators.windowing.buffers.{HeapWindowBuffer, PreAggregatingHeapWindowBuffer} import org.apache.flink.streaming.runtime.operators.windowing.{EvictingWindowOperator, WindowOperator, AccumulatingProcessingTimeWindowOperator, AggregatingProcessingTimeWindowOperator} import org.apache.flink.streaming.util.StreamingMultipleProgramsTestBase import org.apache.flink.util.Collector @@ -69,11 +69,11 @@ class WindowTranslationTest extends StreamingMultipleProgramsTestBase { .window(SlidingTimeWindows.of( Time.of(1, TimeUnit.SECONDS), Time.of(100, TimeUnit.MILLISECONDS))) - .apply(new WindowFunction[(String, Int), (String, Int), Tuple, TimeWindow]() { + .apply(new WindowFunction[Iterable[(String, Int)], (String, Int), Tuple, TimeWindow]() { def apply( key: Tuple, window: TimeWindow, - values: java.lang.Iterable[(String, Int)], + values: Iterable[(String, Int)], out: Collector[(String, Int)]) { } }) @@ -106,23 +106,23 @@ class WindowTranslationTest extends StreamingMultipleProgramsTestBase { val operator1 = transform1.getOperator - assertTrue(operator1.isInstanceOf[WindowOperator[_, _, _, _]]) - val winOperator1 = operator1.asInstanceOf[WindowOperator[_, _, _, _]] + assertTrue(operator1.isInstanceOf[WindowOperator[_, _, _, _, _]]) + val winOperator1 = operator1.asInstanceOf[WindowOperator[_, _, _, _, _]] assertTrue(winOperator1.getTrigger.isInstanceOf[CountTrigger[_]]) assertTrue(winOperator1.getWindowAssigner.isInstanceOf[SlidingTimeWindows]) assertTrue( - winOperator1.getWindowBufferFactory.isInstanceOf[PreAggregatingHeapWindowBuffer.Factory[_]]) + winOperator1.getStateDescriptor.isInstanceOf[ReducingStateDescriptor[_]]) val window2 = source .keyBy(0) .window(TumblingTimeWindows.of(Time.of(1, TimeUnit.SECONDS))) .trigger(CountTrigger.of(100)) - .apply(new WindowFunction[(String, Int), (String, Int), Tuple, TimeWindow]() { + .apply(new WindowFunction[Iterable[(String, Int)], (String, Int), Tuple, TimeWindow]() { def apply( tuple: Tuple, window: TimeWindow, - values: java.lang.Iterable[(String, Int)], + values: Iterable[(String, Int)], out: Collector[(String, Int)]) { } }) @@ -131,11 +131,11 @@ class WindowTranslationTest extends StreamingMultipleProgramsTestBase { val operator2 = transform2.getOperator - assertTrue(operator2.isInstanceOf[WindowOperator[_, _, _, _]]) - val winOperator2 = operator2.asInstanceOf[WindowOperator[_, _, _, _]] + assertTrue(operator2.isInstanceOf[WindowOperator[_, _, _, _, _]]) + val winOperator2 = operator2.asInstanceOf[WindowOperator[_, _, _, _, _]] assertTrue(winOperator2.getTrigger.isInstanceOf[CountTrigger[_]]) assertTrue(winOperator2.getWindowAssigner.isInstanceOf[TumblingTimeWindows]) - assertTrue(winOperator2.getWindowBufferFactory.isInstanceOf[HeapWindowBuffer.Factory[_]]) + assertTrue(winOperator2.getStateDescriptor.isInstanceOf[ListStateDescriptor[_]]) } @Test @@ -164,7 +164,7 @@ class WindowTranslationTest extends StreamingMultipleProgramsTestBase { assertTrue(winOperator1.getTrigger.isInstanceOf[ProcessingTimeTrigger]) assertTrue(winOperator1.getEvictor.isInstanceOf[TimeEvictor[_]]) assertTrue(winOperator1.getWindowAssigner.isInstanceOf[SlidingTimeWindows]) - assertTrue(winOperator1.getWindowBufferFactory.isInstanceOf[HeapWindowBuffer.Factory[_]]) + assertTrue(winOperator1.getStateDescriptor.isInstanceOf[ListStateDescriptor[_]]) val window2 = source @@ -172,11 +172,11 @@ class WindowTranslationTest extends StreamingMultipleProgramsTestBase { .window(TumblingTimeWindows.of(Time.of(1, TimeUnit.SECONDS))) .trigger(CountTrigger.of(100)) .evictor(CountEvictor.of(1000)) - .apply(new WindowFunction[(String, Int), (String, Int), Tuple, TimeWindow]() { + .apply(new WindowFunction[Iterable[(String, Int)], (String, Int), Tuple, TimeWindow]() { def apply( tuple: Tuple, window: TimeWindow, - values: java.lang.Iterable[(String, Int)], + values: Iterable[(String, Int)], out: Collector[(String, Int)]) { } }) @@ -190,7 +190,7 @@ class WindowTranslationTest extends StreamingMultipleProgramsTestBase { assertTrue(winOperator2.getTrigger.isInstanceOf[CountTrigger[_]]) assertTrue(winOperator2.getEvictor.isInstanceOf[CountEvictor[_]]) assertTrue(winOperator2.getWindowAssigner.isInstanceOf[TumblingTimeWindows]) - assertTrue(winOperator2.getWindowBufferFactory.isInstanceOf[HeapWindowBuffer.Factory[_]]) + assertTrue(winOperator2.getStateDescriptor.isInstanceOf[ListStateDescriptor[_]]) } @Test @@ -211,7 +211,7 @@ class WindowTranslationTest extends StreamingMultipleProgramsTestBase { def apply( tuple: Tuple, window: TimeWindow, - values: java.lang.Iterable[(String, Int)], + values: (String, Int), out: Collector[(String, Int)]) { } }) @@ -220,12 +220,12 @@ class WindowTranslationTest extends StreamingMultipleProgramsTestBase { val operator1 = transform1.getOperator - assertTrue(operator1.isInstanceOf[WindowOperator[_, _, _, _]]) - val winOperator1 = operator1.asInstanceOf[WindowOperator[_, _, _, _]] + assertTrue(operator1.isInstanceOf[WindowOperator[_, _, _, _, _]]) + val winOperator1 = operator1.asInstanceOf[WindowOperator[_, _, _, _, _]] assertTrue(winOperator1.getTrigger.isInstanceOf[CountTrigger[_]]) assertTrue(winOperator1.getWindowAssigner.isInstanceOf[SlidingTimeWindows]) assertTrue( - winOperator1.getWindowBufferFactory.isInstanceOf[PreAggregatingHeapWindowBuffer.Factory[_]]) + winOperator1.getStateDescriptor.isInstanceOf[ReducingStateDescriptor[_]]) val window2 = source @@ -236,7 +236,7 @@ class WindowTranslationTest extends StreamingMultipleProgramsTestBase { def apply( tuple: Tuple, window: TimeWindow, - values: java.lang.Iterable[(String, Int)], + values: (String, Int), out: Collector[(String, Int)]) { } }) @@ -245,11 +245,11 @@ class WindowTranslationTest extends StreamingMultipleProgramsTestBase { val operator2 = transform2.getOperator - assertTrue(operator2.isInstanceOf[WindowOperator[_, _, _, _]]) - val winOperator2 = operator2.asInstanceOf[WindowOperator[_, _, _, _]] + assertTrue(operator2.isInstanceOf[WindowOperator[_, _, _, _, _]]) + val winOperator2 = operator2.asInstanceOf[WindowOperator[_, _, _, _, _]] assertTrue(winOperator2.getTrigger.isInstanceOf[CountTrigger[_]]) assertTrue(winOperator2.getWindowAssigner.isInstanceOf[TumblingTimeWindows]) assertTrue( - winOperator2.getWindowBufferFactory.isInstanceOf[PreAggregatingHeapWindowBuffer.Factory[_]]) + winOperator2.getStateDescriptor.isInstanceOf[ReducingStateDescriptor[_]]) } } diff --git a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/EventTimeAllWindowCheckpointingITCase.java b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/EventTimeAllWindowCheckpointingITCase.java index 18c1b3c06d9fd..9eca07477719d 100644 --- a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/EventTimeAllWindowCheckpointingITCase.java +++ b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/EventTimeAllWindowCheckpointingITCase.java @@ -18,13 +18,13 @@ package org.apache.flink.test.checkpointing; -import org.apache.flink.api.common.functions.RichReduceFunction; +import org.apache.flink.api.common.functions.ReduceFunction; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.api.java.tuple.Tuple4; import org.apache.flink.configuration.ConfigConstants; import org.apache.flink.configuration.Configuration; -import org.apache.flink.runtime.state.CheckpointListener; import org.apache.flink.streaming.api.TimeCharacteristic; +import org.apache.flink.runtime.state.CheckpointListener; import org.apache.flink.streaming.api.checkpoint.Checkpointed; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.flink.streaming.api.functions.sink.RichSinkFunction; @@ -105,7 +105,7 @@ public void testTumblingTimeWindow() { NUM_ELEMENTS_PER_KEY / 3)) .rebalance() .timeWindowAll(Time.of(WINDOW_SIZE, MILLISECONDS)) - .apply(new RichAllWindowFunction, Tuple4, TimeWindow>() { + .apply(new RichAllWindowFunction>, Tuple4, TimeWindow>() { private boolean open = false; @@ -167,7 +167,7 @@ public void testSlidingTimeWindow() { .addSource(new FailingSource(NUM_KEYS, NUM_ELEMENTS_PER_KEY, NUM_ELEMENTS_PER_KEY / 3)) .rebalance() .timeWindowAll(Time.of(WINDOW_SIZE, MILLISECONDS), Time.of(WINDOW_SLIDE, MILLISECONDS)) - .apply(new RichAllWindowFunction, Tuple4, TimeWindow>() { + .apply(new RichAllWindowFunction>, Tuple4, TimeWindow>() { private boolean open = false; @@ -231,23 +231,13 @@ public void testPreAggregatedTumblingTimeWindow() { .rebalance() .timeWindowAll(Time.of(WINDOW_SIZE, MILLISECONDS)) .apply( - new RichReduceFunction>() { - - private boolean open = false; - - @Override - public void open(Configuration parameters) { - assertEquals(1, getRuntimeContext().getNumberOfParallelSubtasks()); - open = true; - } + new ReduceFunction>() { @Override public Tuple2 reduce( Tuple2 a, Tuple2 b) { - // validate that the function has been opened properly - assertTrue(open); return new Tuple2<>(a.f0, new IntType(a.f1.value + b.f1.value)); } }, @@ -264,20 +254,13 @@ public void open(Configuration parameters) { @Override public void apply( TimeWindow window, - Iterable> values, + Tuple2 input, Collector> out) { // validate that the function has been opened properly assertTrue(open); - int sum = 0; - long key = -1; - - for (Tuple2 value : values) { - sum += value.f1.value; - key = value.f0; - } - out.collect(new Tuple4<>(key, window.getStart(), window.getEnd(), new IntType(sum))); + out.collect(new Tuple4<>(input.f0, window.getStart(), window.getEnd(), input.f1)); } }) .addSink(new ValidatingSink(NUM_KEYS, NUM_ELEMENTS_PER_KEY / WINDOW_SIZE)).setParallelism(1); @@ -317,23 +300,13 @@ public void testPreAggregatedSlidingTimeWindow() { .timeWindowAll(Time.of(WINDOW_SIZE, MILLISECONDS), Time.of(WINDOW_SLIDE, MILLISECONDS)) .apply( - new RichReduceFunction>() { - - private boolean open = false; - - @Override - public void open(Configuration parameters) { - assertEquals(1, getRuntimeContext().getNumberOfParallelSubtasks()); - open = true; - } + new ReduceFunction>() { @Override public Tuple2 reduce( Tuple2 a, Tuple2 b) { - // validate that the function has been opened properly - assertTrue(open); return new Tuple2<>(a.f0, new IntType(a.f1.value + b.f1.value)); } }, @@ -350,20 +323,13 @@ public void open(Configuration parameters) { @Override public void apply( TimeWindow window, - Iterable> values, + Tuple2 input, Collector> out) { // validate that the function has been opened properly assertTrue(open); - int sum = 0; - long key = -1; - - for (Tuple2 value : values) { - sum += value.f1.value; - key = value.f0; - } - out.collect(new Tuple4<>(key, window.getStart(), window.getEnd(), new IntType(sum))); + out.collect(new Tuple4<>(input.f0, window.getStart(), window.getEnd(), input.f1)); } }) .addSink(new ValidatingSink(NUM_KEYS, NUM_ELEMENTS_PER_KEY / WINDOW_SLIDE)).setParallelism(1); diff --git a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/EventTimeWindowCheckpointingITCase.java b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/EventTimeWindowCheckpointingITCase.java index 7a1a879c72769..5886982d17caa 100644 --- a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/EventTimeWindowCheckpointingITCase.java +++ b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/EventTimeWindowCheckpointingITCase.java @@ -18,15 +18,18 @@ package org.apache.flink.test.checkpointing; -import org.apache.flink.api.common.functions.RichReduceFunction; +import org.apache.flink.api.common.functions.ReduceFunction; import org.apache.flink.api.common.state.OperatorState; import org.apache.flink.api.java.tuple.Tuple; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.api.java.tuple.Tuple4; import org.apache.flink.configuration.ConfigConstants; import org.apache.flink.configuration.Configuration; -import org.apache.flink.runtime.state.CheckpointListener; +import org.apache.flink.runtime.state.AbstractStateBackend; +import org.apache.flink.runtime.state.filesystem.FsStateBackend; +import org.apache.flink.runtime.state.memory.MemoryStateBackend; import org.apache.flink.streaming.api.TimeCharacteristic; +import org.apache.flink.runtime.state.CheckpointListener; import org.apache.flink.streaming.api.checkpoint.Checkpointed; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.flink.streaming.api.functions.sink.RichSinkFunction; @@ -40,9 +43,17 @@ import org.apache.flink.util.Collector; import org.apache.flink.util.TestLogger; import org.junit.AfterClass; +import org.junit.Before; import org.junit.BeforeClass; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import java.io.IOException; +import java.util.Arrays; +import java.util.Collection; import java.util.HashMap; import static java.util.concurrent.TimeUnit.MILLISECONDS; @@ -55,12 +66,22 @@ * of the emitted windows are deterministic. */ @SuppressWarnings("serial") +@RunWith(Parameterized.class) public class EventTimeWindowCheckpointingITCase extends TestLogger { private static final int PARALLELISM = 4; private static ForkableFlinkMiniCluster cluster; + @Rule + public TemporaryFolder tempFolder = new TemporaryFolder(); + + private StateBackendEnum stateBackendEnum; + private AbstractStateBackend stateBackend; + + public EventTimeWindowCheckpointingITCase(StateBackendEnum stateBackendEnum) { + this.stateBackendEnum = stateBackendEnum; + } @BeforeClass public static void startTestCluster() { @@ -81,6 +102,19 @@ public static void stopTestCluster() { } } + @Before + public void initStateBackend() throws IOException { + switch (stateBackendEnum) { + case MEM: + this.stateBackend = new MemoryStateBackend(); + break; + case FILE: + String backups = tempFolder.newFolder().getAbsolutePath(); + this.stateBackend = new FsStateBackend("file://" + backups); + break; + } + } + // ------------------------------------------------------------------------ @Test @@ -99,13 +133,14 @@ public void testTumblingTimeWindow() { env.enableCheckpointing(100); env.setNumberOfExecutionRetries(3); env.getConfig().disableSysoutLogging(); + env.setStateBackend(this.stateBackend); env .addSource(new FailingSource(NUM_KEYS, NUM_ELEMENTS_PER_KEY, NUM_ELEMENTS_PER_KEY / 3)) .rebalance() .keyBy(0) .timeWindow(Time.of(WINDOW_SIZE, MILLISECONDS)) - .apply(new RichWindowFunction, Tuple4, Tuple, TimeWindow>() { + .apply(new RichWindowFunction>, Tuple4, Tuple, TimeWindow>() { private boolean open = false; @@ -162,13 +197,14 @@ public void testTumblingTimeWindowWithKVState() { env.enableCheckpointing(100); env.setNumberOfExecutionRetries(3); env.getConfig().disableSysoutLogging(); + env.setStateBackend(this.stateBackend); env .addSource(new FailingSource(NUM_KEYS, NUM_ELEMENTS_PER_KEY, NUM_ELEMENTS_PER_KEY / 3)) .rebalance() .keyBy(0) .timeWindow(Time.of(WINDOW_SIZE, MILLISECONDS)) - .apply(new RichWindowFunction, Tuple4, Tuple, TimeWindow>() { + .apply(new RichWindowFunction>, Tuple4, Tuple, TimeWindow>() { private boolean open = false; @@ -229,13 +265,14 @@ public void testSlidingTimeWindow() { env.enableCheckpointing(100); env.setNumberOfExecutionRetries(3); env.getConfig().disableSysoutLogging(); + env.setStateBackend(this.stateBackend); env .addSource(new FailingSource(NUM_KEYS, NUM_ELEMENTS_PER_KEY, NUM_ELEMENTS_PER_KEY / 3)) .rebalance() .keyBy(0) .timeWindow(Time.of(WINDOW_SIZE, MILLISECONDS), Time.of(WINDOW_SLIDE, MILLISECONDS)) - .apply(new RichWindowFunction, Tuple4, Tuple, TimeWindow>() { + .apply(new RichWindowFunction>, Tuple4, Tuple, TimeWindow>() { private boolean open = false; @@ -292,6 +329,7 @@ public void testPreAggregatedTumblingTimeWindow() { env.enableCheckpointing(100); env.setNumberOfExecutionRetries(3); env.getConfig().disableSysoutLogging(); + env.setStateBackend(this.stateBackend); env .addSource(new FailingSource(NUM_KEYS, NUM_ELEMENTS_PER_KEY, NUM_ELEMENTS_PER_KEY / 3)) @@ -299,23 +337,12 @@ public void testPreAggregatedTumblingTimeWindow() { .keyBy(0) .timeWindow(Time.of(WINDOW_SIZE, MILLISECONDS)) .apply( - new RichReduceFunction>() { - - private boolean open = false; - - @Override - public void open(Configuration parameters) { - assertEquals(PARALLELISM, getRuntimeContext().getNumberOfParallelSubtasks()); - open = true; - } + new ReduceFunction>() { @Override public Tuple2 reduce( Tuple2 a, Tuple2 b) { - - // validate that the function has been opened properly - assertTrue(open); return new Tuple2<>(a.f0, new IntType(a.f1.value + b.f1.value)); } }, @@ -333,20 +360,13 @@ public void open(Configuration parameters) { public void apply( Tuple tuple, TimeWindow window, - Iterable> values, + Tuple2 input, Collector> out) { // validate that the function has been opened properly assertTrue(open); - int sum = 0; - long key = -1; - - for (Tuple2 value : values) { - sum += value.f1.value; - key = value.f0; - } - out.collect(new Tuple4<>(key, window.getStart(), window.getEnd(), new IntType(sum))); + out.collect(new Tuple4<>(input.f0, window.getStart(), window.getEnd(), input.f1)); } }) .addSink(new ValidatingSink(NUM_KEYS, NUM_ELEMENTS_PER_KEY / WINDOW_SIZE)).setParallelism(1); @@ -377,6 +397,7 @@ public void testPreAggregatedSlidingTimeWindow() { env.enableCheckpointing(100); env.setNumberOfExecutionRetries(3); env.getConfig().disableSysoutLogging(); + env.setStateBackend(this.stateBackend); env .addSource(new FailingSource(NUM_KEYS, NUM_ELEMENTS_PER_KEY, NUM_ELEMENTS_PER_KEY / 3)) @@ -384,15 +405,7 @@ public void testPreAggregatedSlidingTimeWindow() { .keyBy(0) .timeWindow(Time.of(WINDOW_SIZE, MILLISECONDS), Time.of(WINDOW_SLIDE, MILLISECONDS)) .apply( - new RichReduceFunction>() { - - private boolean open = false; - - @Override - public void open(Configuration parameters) { - assertEquals(PARALLELISM, getRuntimeContext().getNumberOfParallelSubtasks()); - open = true; - } + new ReduceFunction>() { @Override public Tuple2 reduce( @@ -400,7 +413,6 @@ public Tuple2 reduce( Tuple2 b) { // validate that the function has been opened properly - assertTrue(open); return new Tuple2<>(a.f0, new IntType(a.f1.value + b.f1.value)); } }, @@ -418,20 +430,13 @@ public void open(Configuration parameters) { public void apply( Tuple tuple, TimeWindow window, - Iterable> values, + Tuple2 input, Collector> out) { // validate that the function has been opened properly assertTrue(open); - int sum = 0; - long key = -1; - - for (Tuple2 value : values) { - sum += value.f1.value; - key = value.f0; - } - out.collect(new Tuple4<>(key, window.getStart(), window.getEnd(), new IntType(sum))); + out.collect(new Tuple4<>(input.f0, window.getStart(), window.getEnd(), input.f1)); } }) .addSink(new ValidatingSink(NUM_KEYS, NUM_ELEMENTS_PER_KEY / WINDOW_SLIDE)).setParallelism(1); @@ -583,7 +588,7 @@ public void close() throws Exception { } } } - assertTrue("The source must see all expected windows.", seenAll); + assertTrue("The sink must see all expected windows.", seenAll); } @Override @@ -723,6 +728,25 @@ public void restoreState(HashMap state) { } } + // ------------------------------------------------------------------------ + // Parametrization for testing with different state backends + // ------------------------------------------------------------------------ + + + @Parameterized.Parameters(name = "StateBackend = {0}") + @SuppressWarnings("unchecked,rawtypes") + public static Collection parameters(){ + return Arrays.asList(new Object[][] { + {StateBackendEnum.MEM}, + {StateBackendEnum.FILE}, + } + ); + } + + private enum StateBackendEnum { + MEM, FILE, DB, ROCKSDB + } + // ------------------------------------------------------------------------ // Utilities diff --git a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/WindowCheckpointingITCase.java b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/WindowCheckpointingITCase.java index 8d59975bb4686..c9286ce6a1acc 100644 --- a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/WindowCheckpointingITCase.java +++ b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/WindowCheckpointingITCase.java @@ -19,15 +19,15 @@ package org.apache.flink.test.checkpointing; import org.apache.flink.api.common.functions.MapFunction; -import org.apache.flink.api.common.functions.RichReduceFunction; +import org.apache.flink.api.common.functions.ReduceFunction; import org.apache.flink.api.java.tuple.Tuple; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.client.program.ProgramInvocationException; import org.apache.flink.configuration.ConfigConstants; import org.apache.flink.configuration.Configuration; import org.apache.flink.runtime.client.JobExecutionException; -import org.apache.flink.runtime.state.CheckpointListener; import org.apache.flink.streaming.api.TimeCharacteristic; +import org.apache.flink.runtime.state.CheckpointListener; import org.apache.flink.streaming.api.checkpoint.Checkpointed; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.flink.streaming.api.functions.sink.RichSinkFunction; @@ -117,7 +117,7 @@ public void testTumblingProcessingTimeWindow() { .rebalance() .keyBy(0) .timeWindow(Time.of(100, MILLISECONDS)) - .apply(new RichWindowFunction, Tuple2, Tuple, TimeWindow>() { + .apply(new RichWindowFunction>, Tuple2, Tuple, TimeWindow>() { private boolean open = false; @@ -175,7 +175,7 @@ public void testSlidingProcessingTimeWindow() { .rebalance() .keyBy(0) .timeWindow(Time.of(150, MILLISECONDS), Time.of(50, MILLISECONDS)) - .apply(new RichWindowFunction, Tuple2, Tuple, TimeWindow>() { + .apply(new RichWindowFunction>, Tuple2, Tuple, TimeWindow>() { private boolean open = false; @@ -240,23 +240,12 @@ public Tuple2 map(Tuple2 value) { .rebalance() .keyBy(0) .timeWindow(Time.of(100, MILLISECONDS)) - .reduce(new RichReduceFunction>() { - - private boolean open = false; - - @Override - public void open(Configuration parameters) { - assertEquals(PARALLELISM, getRuntimeContext().getNumberOfParallelSubtasks()); - open = true; - } + .reduce(new ReduceFunction>() { @Override public Tuple2 reduce( Tuple2 a, Tuple2 b) { - - // validate that the function has been opened properly - assertTrue(open); return new Tuple2<>(a.f0, new IntType(1)); } }) @@ -299,23 +288,11 @@ public Tuple2 map(Tuple2 value) { .rebalance() .keyBy(0) .timeWindow(Time.of(150, MILLISECONDS), Time.of(50, MILLISECONDS)) - .reduce(new RichReduceFunction>() { - - private boolean open = false; - - @Override - public void open(Configuration parameters) { - assertEquals(PARALLELISM, getRuntimeContext().getNumberOfParallelSubtasks()); - open = true; - } - + .reduce(new ReduceFunction>() { @Override public Tuple2 reduce( Tuple2 a, Tuple2 b) { - - // validate that the function has been opened properly - assertTrue(open); return new Tuple2<>(a.f0, new IntType(1)); } }) From afcc0ec4e3c90b24f9811551a8d76efeecdf05da Mon Sep 17 00:00:00 2001 From: Aljoscha Krettek Date: Thu, 21 Jan 2016 10:56:47 +0100 Subject: [PATCH 3/3] [FLINK-3278] Add Partitioned State Backend Based on RocksDB --- .../flink-statebackend-rocksdb/pom.xml | 71 ++++ .../streaming/state/AbstractRocksDBState.java | 372 ++++++++++++++++++ .../streaming/state/RocksDBListState.java | 183 +++++++++ .../streaming/state/RocksDBReducingState.java | 190 +++++++++ .../streaming/state/RocksDBStateBackend.java | 127 ++++++ .../streaming/state/RocksDBValueState.java | 156 ++++++++ .../state/RocksDBStateBackendTest.java | 53 +++ .../src/test/resources/log4j-test.properties | 27 ++ .../src/test/resources/log4j.properties | 27 ++ .../src/test/resources/logback-test.xml | 30 ++ flink-contrib/pom.xml | 1 + .../flink/util/ExternalProcessRunner.java | 233 +++++++++++ .../apache/flink/util/HDFSCopyFromLocal.java | 48 +++ .../apache/flink/util/HDFSCopyToLocal.java | 49 +++ .../flink/util/ExternalProcessRunnerTest.java | 98 +++++ flink-tests/pom.xml | 7 + .../EventTimeWindowCheckpointingITCase.java | 9 + 17 files changed, 1681 insertions(+) create mode 100644 flink-contrib/flink-statebackend-rocksdb/pom.xml create mode 100644 flink-contrib/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/AbstractRocksDBState.java create mode 100644 flink-contrib/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBListState.java create mode 100644 flink-contrib/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBReducingState.java create mode 100644 flink-contrib/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBStateBackend.java create mode 100644 flink-contrib/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBValueState.java create mode 100644 flink-contrib/flink-statebackend-rocksdb/src/test/java/org/apache/flink/contrib/streaming/state/RocksDBStateBackendTest.java create mode 100644 flink-contrib/flink-statebackend-rocksdb/src/test/resources/log4j-test.properties create mode 100644 flink-contrib/flink-statebackend-rocksdb/src/test/resources/log4j.properties create mode 100644 flink-contrib/flink-statebackend-rocksdb/src/test/resources/logback-test.xml create mode 100644 flink-core/src/main/java/org/apache/flink/util/ExternalProcessRunner.java create mode 100644 flink-core/src/main/java/org/apache/flink/util/HDFSCopyFromLocal.java create mode 100644 flink-core/src/main/java/org/apache/flink/util/HDFSCopyToLocal.java create mode 100644 flink-core/src/test/java/org/apache/flink/util/ExternalProcessRunnerTest.java diff --git a/flink-contrib/flink-statebackend-rocksdb/pom.xml b/flink-contrib/flink-statebackend-rocksdb/pom.xml new file mode 100644 index 0000000000000..999c496982978 --- /dev/null +++ b/flink-contrib/flink-statebackend-rocksdb/pom.xml @@ -0,0 +1,71 @@ + + + + + 4.0.0 + + + org.apache.flink + flink-contrib + 1.0-SNAPSHOT + .. + + + flink-statebackend-rocksdb_2.10 + flink-statebackend-rocksdb + + jar + + + + org.apache.flink + flink-streaming-java_2.10 + ${project.version} + + + org.apache.flink + flink-clients_2.10 + ${project.version} + + + com.google.guava + guava + ${guava.version} + + + org.rocksdb + rocksdbjni + 4.1.0 + + + + org.apache.flink + flink-runtime_2.10 + ${project.version} + test-jar + test + + + + + diff --git a/flink-contrib/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/AbstractRocksDBState.java b/flink-contrib/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/AbstractRocksDBState.java new file mode 100644 index 0000000000000..6dbe16c2ab708 --- /dev/null +++ b/flink-contrib/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/AbstractRocksDBState.java @@ -0,0 +1,372 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.flink.contrib.streaming.state; + +import org.apache.commons.io.FileUtils; +import org.apache.flink.api.common.state.State; +import org.apache.flink.api.common.state.StateDescriptor; +import org.apache.flink.api.common.typeutils.TypeSerializer; +import org.apache.flink.core.memory.DataOutputView; +import org.apache.flink.core.memory.DataOutputViewStreamWrapper; +import org.apache.flink.runtime.state.AbstractStateBackend; +import org.apache.flink.runtime.state.KvState; +import org.apache.flink.runtime.state.KvStateSnapshot; +import org.apache.flink.util.HDFSCopyFromLocal; +import org.apache.flink.util.HDFSCopyToLocal; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.rocksdb.BackupEngine; +import org.rocksdb.BackupableDBOptions; +import org.rocksdb.Env; +import org.rocksdb.Options; +import org.rocksdb.RestoreOptions; +import org.rocksdb.RocksDB; +import org.rocksdb.RocksDBException; +import org.rocksdb.StringAppendOperator; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.IOException; +import java.net.URI; + +import static java.util.Objects.requireNonNull; + +/** + * Base class for {@link State} implementations that store state in a RocksDB database. + * + *

This base class is responsible for setting up the RocksDB database, for + * checkpointing/restoring the database and for disposal in the {@link #dispose()} method. The + * concrete subclasses just use the RocksDB handle to store/retrieve state. + * + * @param The type of the key. + * @param The type of the namespace. + * @param The type of {@link State}. + * @param The type of {@link StateDescriptor}. + * @param The type of the backend that snapshots this key/value state. + */ +public abstract class AbstractRocksDBState, Backend extends AbstractStateBackend> + implements KvState, State { + + private static final Logger LOG = LoggerFactory.getLogger(AbstractRocksDBState.class); + + /** Serializer for the keys */ + protected final TypeSerializer keySerializer; + + /** Serializer for the namespace */ + protected final TypeSerializer namespaceSerializer; + + /** The current key, which the next value methods will refer to */ + protected K currentKey; + + /** The current namespace, which the next value methods will refer to */ + protected N currentNamespace; + + /** Store it so that we can clean up in dispose() */ + protected final File dbPath; + + protected final String checkpointPath; + + /** Our RocksDB instance */ + protected final RocksDB db; + + /** + * Creates a new RocksDB backed state. + * + * @param keySerializer The serializer for the keys. + * @param namespaceSerializer The serializer for the namespace. + * @param dbPath The path on the local system where RocksDB data should be stored. + */ + protected AbstractRocksDBState(TypeSerializer keySerializer, + TypeSerializer namespaceSerializer, + File dbPath, + String checkpointPath) { + this.keySerializer = requireNonNull(keySerializer); + this.namespaceSerializer = namespaceSerializer; + this.dbPath = dbPath; + this.checkpointPath = checkpointPath; + + RocksDB.loadLibrary(); + + Options options = new Options().setCreateIfMissing(true); + options.setMergeOperator(new StringAppendOperator()); + + if (!dbPath.exists()) { + if (!dbPath.mkdirs()) { + throw new RuntimeException("Could not create RocksDB data directory."); + } + } + + // clean it, this will remove the last part of the path but RocksDB will recreate it + try { + File db = new File(dbPath, "db"); + LOG.warn("Deleting already existing db directory {}.", db); + FileUtils.deleteDirectory(db); + } catch (IOException e) { + throw new RuntimeException("Error cleaning RocksDB data directory.", e); + } + + try { + db = RocksDB.open(options, new File(dbPath, "db").getAbsolutePath()); + } catch (RocksDBException e) { + throw new RuntimeException("Error while opening RocksDB instance.", e); + } + + options.dispose(); + + } + + /** + * Creates a new RocksDB backed state and restores from the given backup directory. After + * restoring the backup directory is deleted. + * + * @param keySerializer The serializer for the keys. + * @param namespaceSerializer The serializer for the namespace. + * @param dbPath The path on the local system where RocksDB data should be stored. + * @param restorePath The path to a backup directory from which to restore RocksDb database. + */ + protected AbstractRocksDBState(TypeSerializer keySerializer, + TypeSerializer namespaceSerializer, + File dbPath, + String checkpointPath, + String restorePath) { + + RocksDB.loadLibrary(); + + try { + BackupEngine backupEngine = BackupEngine.open(Env.getDefault(), new BackupableDBOptions(restorePath + "/")); + backupEngine.restoreDbFromLatestBackup(new File(dbPath, "db").getAbsolutePath(), new File(dbPath, "db").getAbsolutePath(), new RestoreOptions(true)); + FileUtils.deleteDirectory(new File(restorePath)); + } catch (RocksDBException|IOException|IllegalArgumentException e) { + throw new RuntimeException("Error while restoring RocksDB state from " + restorePath, e); + } + + this.keySerializer = requireNonNull(keySerializer); + this.namespaceSerializer = namespaceSerializer; + this.dbPath = dbPath; + this.checkpointPath = checkpointPath; + + Options options = new Options().setCreateIfMissing(true); + options.setMergeOperator(new StringAppendOperator()); + + if (!dbPath.exists()) { + if (!dbPath.mkdirs()) { + throw new RuntimeException("Could not create RocksDB data directory."); + } + } + + try { + db = RocksDB.open(options, new File(dbPath, "db").getAbsolutePath()); + } catch (RocksDBException e) { + throw new RuntimeException("Error while opening RocksDB instance.", e); + } + + options.dispose(); + } + + // ------------------------------------------------------------------------ + + @Override + final public void clear() { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + DataOutputViewStreamWrapper out = new DataOutputViewStreamWrapper(baos); + try { + writeKeyAndNamespace(out); + byte[] key = baos.toByteArray(); + db.remove(key); + } catch (IOException|RocksDBException e) { + throw new RuntimeException("Error while removing entry from RocksDB", e); + } + } + + protected void writeKeyAndNamespace(DataOutputView out) throws IOException { + keySerializer.serialize(currentKey, out); + out.writeByte(42); + namespaceSerializer.serialize(currentNamespace, out); + } + + @Override + final public void setCurrentKey(K currentKey) { + this.currentKey = currentKey; + } + + @Override + final public void setCurrentNamespace(N namespace) { + this.currentNamespace = namespace; + } + + protected abstract KvStateSnapshot createRocksDBSnapshot(URI backupUri, long checkpointId); + + @Override + final public KvStateSnapshot snapshot( + long checkpointId, + long timestamp) throws Exception { + boolean success = false; + + final File localBackupPath = new File(dbPath, "backup-" + checkpointId); + final URI backupUri = new URI(checkpointPath + "/chk-" + checkpointId); + + try { + if (!localBackupPath.exists()) { + if (!localBackupPath.mkdirs()) { + throw new RuntimeException("Could not create local backup path " + localBackupPath); + } + } + + BackupEngine backupEngine = BackupEngine.open(Env.getDefault(), + new BackupableDBOptions(localBackupPath.getAbsolutePath())); + + backupEngine.createNewBackup(db); + + HDFSCopyFromLocal.copyFromLocal(localBackupPath, backupUri); + KvStateSnapshot result = createRocksDBSnapshot(backupUri, checkpointId); + success = true; + return result; + } finally { + FileUtils.deleteDirectory(localBackupPath); + if (!success) { + FileSystem fs = FileSystem.get(backupUri, new Configuration()); + fs.delete(new Path(backupUri), true); + } + } + } + + @Override + final public void dispose() { + db.dispose(); + try { + FileUtils.deleteDirectory(dbPath); + } catch (IOException e) { + throw new RuntimeException("Error disposing RocksDB data directory.", e); + } + } + + public static abstract class AbstractRocksDBSnapshot, Backend extends AbstractStateBackend> implements KvStateSnapshot { + private static final long serialVersionUID = 1L; + + private static final Logger LOG = LoggerFactory.getLogger(AbstractRocksDBSnapshot.class); + + // ------------------------------------------------------------------------ + // Ctor parameters for RocksDB state + // ------------------------------------------------------------------------ + + /** Store it so that we can clean up in dispose() */ + protected final File dbPath; + + /** Where we should put RocksDB backups */ + protected final String checkpointPath; + + // ------------------------------------------------------------------------ + // Info about this checkpoint + // ------------------------------------------------------------------------ + + protected final URI backupUri; + + protected long checkpointId; + + // ------------------------------------------------------------------------ + // For sanity checks + // ------------------------------------------------------------------------ + + /** Key serializer */ + protected final TypeSerializer keySerializer; + + /** Namespace serializer */ + protected final TypeSerializer namespaceSerializer; + + /** Hash of the StateDescriptor, for sanity checks */ + protected final SD stateDesc; + + public AbstractRocksDBSnapshot(File dbPath, + String checkpointPath, + URI backupUri, + long checkpointId, + TypeSerializer keySerializer, + TypeSerializer namespaceSerializer, + SD stateDesc) { + this.dbPath = dbPath; + this.checkpointPath = checkpointPath; + this.backupUri = backupUri; + this.checkpointId = checkpointId; + + this.stateDesc = stateDesc; + this.keySerializer = keySerializer; + this.namespaceSerializer = namespaceSerializer; + } + + protected abstract KvState createRocksDBState(TypeSerializer keySerializer, + TypeSerializer namespaceSerializer, + SD stateDesc, + File dbPath, + String backupPath, + String restorePath) throws Exception; + + @Override + public final KvState restoreState( + Backend stateBackend, + TypeSerializer keySerializer, + ClassLoader classLoader, + long recoveryTimestamp) throws Exception { + + // validity checks + if (!this.keySerializer.equals(keySerializer)) { + throw new IllegalArgumentException( + "Cannot restore the state from the snapshot with the given serializers. " + + "State (K/V) was serialized with " + + "(" + keySerializer + ") " + + "now is (" + keySerializer + ")"); + } + + if (!dbPath.exists()) { + if (!dbPath.mkdirs()) { + throw new RuntimeException("Could not create RocksDB base path " + dbPath); + } + } + + FileSystem fs = FileSystem.get(backupUri, new Configuration()); + + final File localBackupPath = new File(dbPath, "chk-" + checkpointId); + + if (localBackupPath.exists()) { + try { + LOG.warn("Deleting already existing local backup directory {}.", localBackupPath); + FileUtils.deleteDirectory(localBackupPath); + } catch (IOException e) { + throw new RuntimeException("Error cleaning RocksDB local backup directory.", e); + } + } + + HDFSCopyToLocal.copyToLocal(backupUri, dbPath); + return createRocksDBState(keySerializer, namespaceSerializer, stateDesc, dbPath, checkpointPath, localBackupPath.getAbsolutePath()); + } + + @Override + public final void discardState() throws Exception { + FileSystem fs = FileSystem.get(backupUri, new Configuration()); + fs.delete(new Path(backupUri), true); + } + + @Override + public final long getStateSize() throws Exception { + return 0; + } + } +} + diff --git a/flink-contrib/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBListState.java b/flink-contrib/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBListState.java new file mode 100644 index 0000000000000..e97e65de96099 --- /dev/null +++ b/flink-contrib/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBListState.java @@ -0,0 +1,183 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.flink.contrib.streaming.state; + +import org.apache.flink.api.common.state.ListState; +import org.apache.flink.api.common.state.ListStateDescriptor; +import org.apache.flink.api.common.typeutils.TypeSerializer; +import org.apache.flink.core.memory.DataInputViewStreamWrapper; +import org.apache.flink.core.memory.DataOutputViewStreamWrapper; +import org.apache.flink.runtime.state.AbstractStateBackend; +import org.apache.flink.runtime.state.KvState; +import org.apache.flink.runtime.state.KvStateSnapshot; +import org.rocksdb.RocksDBException; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.IOException; +import java.net.URI; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import static java.util.Objects.requireNonNull; + +/** + * {@link ListState} implementation that stores state in RocksDB. + * + * @param The type of the key. + * @param The type of the namespace. + * @param The type of the values in the list state. + * @param The type of the backend that snapshots this key/value state. + */ +public class RocksDBListState + extends AbstractRocksDBState, ListStateDescriptor, Backend> + implements ListState { + + /** Serializer for the values */ + private final TypeSerializer valueSerializer; + + /** This holds the name of the state and can create an initial default value for the state. */ + protected final ListStateDescriptor stateDesc; + + /** + * Creates a new {@code RocksDBListState}. + * + * @param keySerializer The serializer for the keys. + * @param namespaceSerializer The serializer for the namespace. + * @param stateDesc The state identifier for the state. This contains name + * and can create a default state value. + * @param dbPath The path on the local system where RocksDB data should be stored. + */ + protected RocksDBListState(TypeSerializer keySerializer, + TypeSerializer namespaceSerializer, + ListStateDescriptor stateDesc, + File dbPath, + String backupPath) { + super(keySerializer, namespaceSerializer, dbPath, backupPath); + this.stateDesc = requireNonNull(stateDesc); + this.valueSerializer = stateDesc.getSerializer(); + } + + /** + * Creates a new {@code RocksDBListState}. + * + * @param keySerializer The serializer for the keys. + * @param namespaceSerializer The serializer for the namespace. + * @param stateDesc The state identifier for the state. This contains name + * and can create a default state value. + * @param dbPath The path on the local system where RocksDB data should be stored. + */ + protected RocksDBListState(TypeSerializer keySerializer, + TypeSerializer namespaceSerializer, + ListStateDescriptor stateDesc, + File dbPath, + String backupPath, + String restorePath) { + super(keySerializer, namespaceSerializer, dbPath, backupPath, restorePath); + this.stateDesc = requireNonNull(stateDesc); + this.valueSerializer = stateDesc.getSerializer(); + } + + @Override + public Iterable get() { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + DataOutputViewStreamWrapper out = new DataOutputViewStreamWrapper(baos); + try { + writeKeyAndNamespace(out); + byte[] key = baos.toByteArray(); + byte[] valueBytes = db.get(key); + + if (valueBytes == null) { + return Collections.emptyList(); + } + + ByteArrayInputStream bais = new ByteArrayInputStream(valueBytes); + DataInputViewStreamWrapper in = new DataInputViewStreamWrapper(bais); + + List result = new ArrayList<>(); + while (in.available() > 0) { + result.add(valueSerializer.deserialize(in)); + if (in.available() > 0) { + in.readByte(); + } + } + return result; + } catch (IOException|RocksDBException e) { + throw new RuntimeException("Error while retrieving data from RocksDB", e); + } + } + + @Override + public void add(V value) throws IOException { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + DataOutputViewStreamWrapper out = new DataOutputViewStreamWrapper(baos); + try { + writeKeyAndNamespace(out); + byte[] key = baos.toByteArray(); + + baos.reset(); + + valueSerializer.serialize(value, out); + db.merge(key, baos.toByteArray()); + + } catch (Exception e) { + throw new RuntimeException("Error while adding data to RocksDB", e); + } + } + + @Override + protected KvStateSnapshot, ListStateDescriptor, Backend> createRocksDBSnapshot( + URI backupUri, + long checkpointId) { + return new Snapshot<>(dbPath, checkpointPath, backupUri, checkpointId, keySerializer, namespaceSerializer, stateDesc); + } + + private static class Snapshot extends AbstractRocksDBSnapshot, ListStateDescriptor, Backend> { + private static final long serialVersionUID = 1L; + + public Snapshot(File dbPath, + String checkpointPath, + URI backupUri, + long checkpointId, + TypeSerializer keySerializer, + TypeSerializer namespaceSerializer, + ListStateDescriptor stateDesc) { + super(dbPath, + checkpointPath, + backupUri, + checkpointId, + keySerializer, + namespaceSerializer, + stateDesc); + } + + @Override + protected KvState, ListStateDescriptor, Backend> createRocksDBState( + TypeSerializer keySerializer, + TypeSerializer namespaceSerializer, + ListStateDescriptor stateDesc, + File dbPath, + String backupPath, + String restorePath) throws Exception { + return new RocksDBListState<>(keySerializer, namespaceSerializer, stateDesc, dbPath, checkpointPath, restorePath); + } + } +} + diff --git a/flink-contrib/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBReducingState.java b/flink-contrib/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBReducingState.java new file mode 100644 index 0000000000000..eb21c3bab025f --- /dev/null +++ b/flink-contrib/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBReducingState.java @@ -0,0 +1,190 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.flink.contrib.streaming.state; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.flink.api.common.functions.ReduceFunction; +import org.apache.flink.api.common.state.ReducingState; +import org.apache.flink.api.common.state.ReducingStateDescriptor; +import org.apache.flink.api.common.typeutils.TypeSerializer; +import org.apache.flink.core.memory.DataInputViewStreamWrapper; +import org.apache.flink.core.memory.DataOutputViewStreamWrapper; +import org.apache.flink.runtime.state.AbstractStateBackend; +import org.apache.flink.runtime.state.KvState; +import org.apache.flink.runtime.state.KvStateSnapshot; +import org.rocksdb.RocksDBException; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.IOException; +import java.net.URI; + +import static java.util.Objects.requireNonNull; + +/** + * {@link ReducingState} implementation that stores state in RocksDB. + * + * @param The type of the key. + * @param The type of the namespace. + * @param The type of value that the state state stores. + * @param The type of the backend that snapshots this key/value state. + */ +public class RocksDBReducingState + extends AbstractRocksDBState, ReducingStateDescriptor, Backend> + implements ReducingState { + + /** Serializer for the values */ + private final TypeSerializer valueSerializer; + + /** This holds the name of the state and can create an initial default value for the state. */ + protected final ReducingStateDescriptor stateDesc; + + /** User-specified reduce function */ + private final ReduceFunction reduceFunction; + + /** + * Creates a new {@code RocksDBReducingState}. + * + * @param keySerializer The serializer for the keys. + * @param namespaceSerializer The serializer for the namespace. + * @param stateDesc The state identifier for the state. This contains name + * and can create a default state value. + * @param dbPath The path on the local system where RocksDB data should be stored. + */ + protected RocksDBReducingState(TypeSerializer keySerializer, + TypeSerializer namespaceSerializer, + ReducingStateDescriptor stateDesc, + File dbPath, + String backupPath) { + super(keySerializer, namespaceSerializer, dbPath, backupPath); + this.stateDesc = requireNonNull(stateDesc); + this.valueSerializer = stateDesc.getSerializer(); + this.reduceFunction = stateDesc.getReduceFunction(); + } + + protected RocksDBReducingState(TypeSerializer keySerializer, + TypeSerializer namespaceSerializer, + ReducingStateDescriptor stateDesc, + File dbPath, + String backupPath, + String restorePath) { + super(keySerializer, namespaceSerializer, dbPath, backupPath, restorePath); + this.stateDesc = stateDesc; + this.valueSerializer = stateDesc.getSerializer(); + this.reduceFunction = stateDesc.getReduceFunction(); + } + + @Override + public V get() { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + DataOutputViewStreamWrapper out = new DataOutputViewStreamWrapper(baos); + try { + writeKeyAndNamespace(out); + byte[] key = baos.toByteArray(); + byte[] valueBytes = db.get(key); + if (valueBytes == null) { + return null; + } + return valueSerializer.deserialize(new DataInputViewStreamWrapper(new ByteArrayInputStream(valueBytes))); + } catch (IOException|RocksDBException e) { + throw new RuntimeException("Error while retrieving data from RocksDB", e); + } + } + + @Override + public void add(V value) throws IOException { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + DataOutputViewStreamWrapper out = new DataOutputViewStreamWrapper(baos); + try { + writeKeyAndNamespace(out); + byte[] key = baos.toByteArray(); + byte[] valueBytes = db.get(key); + + if (valueBytes == null) { + baos.reset(); + valueSerializer.serialize(value, out); + db.put(key, baos.toByteArray()); + } else { + V oldValue = valueSerializer.deserialize(new DataInputViewStreamWrapper(new ByteArrayInputStream(valueBytes))); + V newValue = reduceFunction.reduce(oldValue, value); + baos.reset(); + valueSerializer.serialize(newValue, out); + db.put(key, baos.toByteArray()); + } + } catch (Exception e) { + throw new RuntimeException("Error while adding data to RocksDB", e); + } + } + + @Override + protected KvStateSnapshot, ReducingStateDescriptor, Backend> createRocksDBSnapshot( + URI backupUri, + long checkpointId) { + return new Snapshot<>(dbPath, checkpointPath, backupUri, checkpointId, keySerializer, namespaceSerializer, stateDesc); + } + + private static class Snapshot extends AbstractRocksDBSnapshot, ReducingStateDescriptor, Backend> { + private static final long serialVersionUID = 1L; + + public Snapshot(File dbPath, + String checkpointPath, + URI backupUri, + long checkpointId, + TypeSerializer keySerializer, + TypeSerializer namespaceSerializer, + ReducingStateDescriptor stateDesc) { + super(dbPath, + checkpointPath, + backupUri, + checkpointId, + keySerializer, + namespaceSerializer, + stateDesc); + } + + @Override + protected KvState, ReducingStateDescriptor, Backend> createRocksDBState( + TypeSerializer keySerializer, + TypeSerializer namespaceSerializer, + ReducingStateDescriptor stateDesc, + File dbPath, + String backupPath, + String restorePath) throws Exception { + return new RocksDBReducingState<>(keySerializer, namespaceSerializer, stateDesc, dbPath, checkpointPath, restorePath); + } + } +} + diff --git a/flink-contrib/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBStateBackend.java b/flink-contrib/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBStateBackend.java new file mode 100644 index 0000000000000..aaaeea491870c --- /dev/null +++ b/flink-contrib/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBStateBackend.java @@ -0,0 +1,127 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.contrib.streaming.state; + +import java.io.File; +import java.io.Serializable; + +import org.apache.flink.api.common.JobID; +import org.apache.flink.api.common.state.ListState; +import org.apache.flink.api.common.state.ListStateDescriptor; +import org.apache.flink.api.common.state.ReducingState; +import org.apache.flink.api.common.state.ReducingStateDescriptor; +import org.apache.flink.api.common.state.ValueState; +import org.apache.flink.api.common.state.ValueStateDescriptor; +import org.apache.flink.api.common.typeutils.TypeSerializer; +import org.apache.flink.runtime.execution.Environment; +import org.apache.flink.runtime.state.AbstractStateBackend; +import org.apache.flink.runtime.state.StateHandle; + +import static java.util.Objects.requireNonNull; + +/** + * + */ +public class RocksDBStateBackend extends AbstractStateBackend { + private static final long serialVersionUID = 1L; + + /** Base path for RocksDB directory. */ + private final String dbBasePath; + + /** The checkpoint directory that we snapshot RocksDB backups to. */ + private final String checkpointDirectory; + + /** Operator identifier that is used to uniqueify the RocksDB storage path. */ + private String operatorIdentifier; + + /** JobID for uniquifying backup paths. */ + private JobID jobId; + + private AbstractStateBackend backingStateBackend; + + public RocksDBStateBackend(String dbBasePath, String checkpointDirectory, AbstractStateBackend backingStateBackend) { + this.dbBasePath = requireNonNull(dbBasePath); + this.checkpointDirectory = requireNonNull(checkpointDirectory); + this.backingStateBackend = requireNonNull(backingStateBackend); + } + + @Override + public void initializeForJob(Environment env, + String operatorIdentifier, + TypeSerializer keySerializer) throws Exception { + super.initializeForJob(env, operatorIdentifier, keySerializer); + this.operatorIdentifier = operatorIdentifier.replace(" ", ""); + backingStateBackend.initializeForJob(env, operatorIdentifier, keySerializer); + this.jobId = env.getJobID(); + } + + @Override + public void disposeAllStateForCurrentJob() throws Exception { + + } + + @Override + public void close() throws Exception { + + } + + private File getDbPath(String stateName) { + return new File(new File(new File(new File(dbBasePath), jobId.toShortString()), operatorIdentifier), stateName); + } + + private String getCheckpointPath(String stateName) { + return checkpointDirectory + "/" + jobId.toShortString() + "/" + operatorIdentifier + "/" + stateName; + } + + @Override + protected ValueState createValueState(TypeSerializer namespaceSerializer, + ValueStateDescriptor stateDesc) throws Exception { + File dbPath = getDbPath(stateDesc.getName()); + String checkpointPath = getCheckpointPath(stateDesc.getName()); + return new RocksDBValueState<>(keySerializer, namespaceSerializer, stateDesc, dbPath, checkpointPath); + } + + @Override + protected ListState createListState(TypeSerializer namespaceSerializer, + ListStateDescriptor stateDesc) throws Exception { + File dbPath = getDbPath(stateDesc.getName()); + String checkpointPath = getCheckpointPath(stateDesc.getName()); + return new RocksDBListState<>(keySerializer, namespaceSerializer, stateDesc, dbPath, checkpointPath); + } + + @Override + protected ReducingState createReducingState(TypeSerializer namespaceSerializer, + ReducingStateDescriptor stateDesc) throws Exception { + File dbPath = getDbPath(stateDesc.getName()); + String checkpointPath = getCheckpointPath(stateDesc.getName()); + return new RocksDBReducingState<>(keySerializer, namespaceSerializer, stateDesc, dbPath, checkpointPath); + } + + @Override + public CheckpointStateOutputStream createCheckpointStateOutputStream(long checkpointID, + long timestamp) throws Exception { + return backingStateBackend.createCheckpointStateOutputStream(checkpointID, timestamp); + } + + @Override + public StateHandle checkpointStateSerializable(S state, + long checkpointID, + long timestamp) throws Exception { + return backingStateBackend.checkpointStateSerializable(state, checkpointID, timestamp); + } +} diff --git a/flink-contrib/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBValueState.java b/flink-contrib/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBValueState.java new file mode 100644 index 0000000000000..8767a8645989c --- /dev/null +++ b/flink-contrib/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBValueState.java @@ -0,0 +1,156 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.flink.contrib.streaming.state; + +import org.apache.flink.api.common.state.ValueState; +import org.apache.flink.api.common.state.ValueStateDescriptor; +import org.apache.flink.api.common.typeutils.TypeSerializer; +import org.apache.flink.core.memory.DataInputViewStreamWrapper; +import org.apache.flink.core.memory.DataOutputViewStreamWrapper; +import org.apache.flink.runtime.state.AbstractStateBackend; +import org.apache.flink.runtime.state.KvState; +import org.apache.flink.runtime.state.KvStateSnapshot; +import org.rocksdb.RocksDBException; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.IOException; +import java.net.URI; + +import static java.util.Objects.requireNonNull; + +/** + * {@link ValueState} implementation that stores state in RocksDB. + * + * @param The type of the key. + * @param The type of the namespace. + * @param The type of value that the state state stores. + * @param The type of the backend that snapshots this key/value state. + */ +public class RocksDBValueState + extends AbstractRocksDBState, ValueStateDescriptor, Backend> + implements ValueState { + + /** Serializer for the values */ + private final TypeSerializer valueSerializer; + + /** This holds the name of the state and can create an initial default value for the state. */ + protected final ValueStateDescriptor stateDesc; + + /** + * Creates a new {@code RocksDBReducingState}. + * + * @param keySerializer The serializer for the keys. + * @param namespaceSerializer The serializer for the namespace. + * @param stateDesc The state identifier for the state. This contains name + * and can create a default state value. + * @param dbPath The path on the local system where RocksDB data should be stored. + */ + protected RocksDBValueState(TypeSerializer keySerializer, + TypeSerializer namespaceSerializer, + ValueStateDescriptor stateDesc, + File dbPath, + String backupPath) { + super(keySerializer, namespaceSerializer, dbPath, backupPath); + this.stateDesc = requireNonNull(stateDesc); + this.valueSerializer = stateDesc.getSerializer(); + } + + protected RocksDBValueState(TypeSerializer keySerializer, + TypeSerializer namespaceSerializer, + ValueStateDescriptor stateDesc, + File dbPath, + String backupPath, + String restorePath) { + super(keySerializer, namespaceSerializer, dbPath, backupPath, restorePath); + this.stateDesc = stateDesc; + this.valueSerializer = stateDesc.getSerializer(); + } + + @Override + public V value() { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + DataOutputViewStreamWrapper out = new DataOutputViewStreamWrapper(baos); + try { + writeKeyAndNamespace(out); + byte[] key = baos.toByteArray(); + byte[] valueBytes = db.get(key); + if (valueBytes == null) { + return stateDesc.getDefaultValue(); + } + return valueSerializer.deserialize(new DataInputViewStreamWrapper(new ByteArrayInputStream(valueBytes))); + } catch (IOException|RocksDBException e) { + throw new RuntimeException("Error while retrieving data from RocksDB", e); + } + } + + @Override + public void update(V value) throws IOException { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + DataOutputViewStreamWrapper out = new DataOutputViewStreamWrapper(baos); + try { + writeKeyAndNamespace(out); + byte[] key = baos.toByteArray(); + baos.reset(); + valueSerializer.serialize(value, out); + db.put(key, baos.toByteArray()); + } catch (Exception e) { + throw new RuntimeException("Error while adding data to RocksDB", e); + } + } + + @Override + protected KvStateSnapshot, ValueStateDescriptor, Backend> createRocksDBSnapshot( + URI backupUri, + long checkpointId) { + return new Snapshot<>(dbPath, checkpointPath, backupUri, checkpointId, keySerializer, namespaceSerializer, stateDesc); + } + + private static class Snapshot extends AbstractRocksDBSnapshot, ValueStateDescriptor, Backend> { + private static final long serialVersionUID = 1L; + + public Snapshot(File dbPath, + String checkpointPath, + URI backupUri, + long checkpointId, + TypeSerializer keySerializer, + TypeSerializer namespaceSerializer, + ValueStateDescriptor stateDesc) { + super(dbPath, + checkpointPath, + backupUri, + checkpointId, + keySerializer, + namespaceSerializer, + stateDesc); + } + + @Override + protected KvState, ValueStateDescriptor, Backend> createRocksDBState( + TypeSerializer keySerializer, + TypeSerializer namespaceSerializer, + ValueStateDescriptor stateDesc, + File dbPath, + String backupPath, + String restorePath) throws Exception { + return new RocksDBValueState<>(keySerializer, namespaceSerializer, stateDesc, dbPath, checkpointPath, restorePath); + } + } +} + diff --git a/flink-contrib/flink-statebackend-rocksdb/src/test/java/org/apache/flink/contrib/streaming/state/RocksDBStateBackendTest.java b/flink-contrib/flink-statebackend-rocksdb/src/test/java/org/apache/flink/contrib/streaming/state/RocksDBStateBackendTest.java new file mode 100644 index 0000000000000..3b3ac31228340 --- /dev/null +++ b/flink-contrib/flink-statebackend-rocksdb/src/test/java/org/apache/flink/contrib/streaming/state/RocksDBStateBackendTest.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.contrib.streaming.state; + +import org.apache.commons.io.FileUtils; +import org.apache.flink.configuration.ConfigConstants; +import org.apache.flink.runtime.state.StateBackendTestBase; +import org.apache.flink.runtime.state.memory.MemoryStateBackend; + +import java.io.File; +import java.io.IOException; +import java.util.UUID; + +/** + * Tests for the partitioned state part of {@link RocksDBStateBackend}. + */ +public class RocksDBStateBackendTest extends StateBackendTestBase { + + private File dbDir; + private File chkDir; + + @Override + protected RocksDBStateBackend getStateBackend() throws IOException { + dbDir = new File(ConfigConstants.DEFAULT_TASK_MANAGER_TMP_PATH, UUID.randomUUID().toString()); + chkDir = new File(ConfigConstants.DEFAULT_TASK_MANAGER_TMP_PATH, UUID.randomUUID().toString()); + + return new RocksDBStateBackend(dbDir.getAbsolutePath(), "file://" + chkDir.getAbsolutePath(), new MemoryStateBackend()); + } + + @Override + protected void cleanup() { + try { + FileUtils.deleteDirectory(dbDir); + FileUtils.deleteDirectory(chkDir); + } catch (IOException ignore) {} + } +} diff --git a/flink-contrib/flink-statebackend-rocksdb/src/test/resources/log4j-test.properties b/flink-contrib/flink-statebackend-rocksdb/src/test/resources/log4j-test.properties new file mode 100644 index 0000000000000..0b686e543bb23 --- /dev/null +++ b/flink-contrib/flink-statebackend-rocksdb/src/test/resources/log4j-test.properties @@ -0,0 +1,27 @@ +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +# Set root logger level to DEBUG and its only appender to A1. +log4j.rootLogger=OFF, A1 + +# A1 is set to be a ConsoleAppender. +log4j.appender.A1=org.apache.log4j.ConsoleAppender + +# A1 uses PatternLayout. +log4j.appender.A1.layout=org.apache.log4j.PatternLayout +log4j.appender.A1.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n \ No newline at end of file diff --git a/flink-contrib/flink-statebackend-rocksdb/src/test/resources/log4j.properties b/flink-contrib/flink-statebackend-rocksdb/src/test/resources/log4j.properties new file mode 100644 index 0000000000000..ed2bbcbbd8167 --- /dev/null +++ b/flink-contrib/flink-statebackend-rocksdb/src/test/resources/log4j.properties @@ -0,0 +1,27 @@ +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +# This file ensures that tests executed from the IDE show log output + +log4j.rootLogger=OFF, console + +# Log all infos in the given file +log4j.appender.console=org.apache.log4j.ConsoleAppender +log4j.appender.console.target = System.err +log4j.appender.console.layout=org.apache.log4j.PatternLayout +log4j.appender.console.layout.ConversionPattern=%d{HH:mm:ss,SSS} %-5p %-60c %x - %m%n \ No newline at end of file diff --git a/flink-contrib/flink-statebackend-rocksdb/src/test/resources/logback-test.xml b/flink-contrib/flink-statebackend-rocksdb/src/test/resources/logback-test.xml new file mode 100644 index 0000000000000..4f56748368989 --- /dev/null +++ b/flink-contrib/flink-statebackend-rocksdb/src/test/resources/logback-test.xml @@ -0,0 +1,30 @@ + + + + + + %d{HH:mm:ss.SSS} [%thread] %-5level %logger{60} %X{sourceThread} - %msg%n + + + + + + + + \ No newline at end of file diff --git a/flink-contrib/pom.xml b/flink-contrib/pom.xml index 82b6211d5818b..76f0f88ec7d17 100644 --- a/flink-contrib/pom.xml +++ b/flink-contrib/pom.xml @@ -43,5 +43,6 @@ under the License. flink-tweet-inputformat flink-operator-stats flink-connector-wikiedits + flink-statebackend-rocksdb diff --git a/flink-core/src/main/java/org/apache/flink/util/ExternalProcessRunner.java b/flink-core/src/main/java/org/apache/flink/util/ExternalProcessRunner.java new file mode 100644 index 0000000000000..8e4725c6c26ce --- /dev/null +++ b/flink-core/src/main/java/org/apache/flink/util/ExternalProcessRunner.java @@ -0,0 +1,233 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.flink.util; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.StringWriter; +import java.lang.management.ManagementFactory; +import java.lang.management.RuntimeMXBean; +import java.lang.reflect.Method; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +/** + * Utility class for running a class in an external process. This will try to find the java + * executable in common places and will use the classpath of the current process as the classpath + * of the new process. + * + *

Attention: The entry point class must be in the classpath of the currently running process, + * otherwise the newly spawned process will not find it and fail. + */ +public class ExternalProcessRunner { + private final String entryPointClassName; + + private final Process process; + + final StringWriter errorOutput = new StringWriter(); + + /** + * Creates a new {@code ProcessRunner} that runs the given class with the given parameters. + * The class must have a "main" method. + */ + public ExternalProcessRunner(String entryPointClassName, String[] parameters) throws IOException { + this.entryPointClassName = entryPointClassName; + + String javaCommand = getJavaCommandPath(); + + List commandList = new ArrayList<>(); + + commandList.add(javaCommand); + commandList.add("-classpath"); + commandList.add(getCurrentClasspath()); + commandList.add(entryPointClassName); + + Collections.addAll(commandList, parameters); + + process = new ProcessBuilder(commandList).start(); + + new PipeForwarder(process.getErrorStream(), errorOutput); + } + + /** + * Get the stderr stream of the process. + */ + public StringWriter getErrorOutput() { + return errorOutput; + } + + /** + * Start the external process, wait for it to finish and return the exit code of that process. + * + *

If this method is interrupted it will destroy the external process and forward the + * {@code InterruptedException}. + */ + public int run() throws Exception { + try { + int returnCode = process.waitFor(); + + if (returnCode != 0) { + // determine whether we failed because of a ClassNotFoundException and forward that + if (getErrorOutput().toString().contains("Error: Could not find or load main class " + entryPointClassName)) { + throw new ClassNotFoundException("Error: Could not find or load main class " + entryPointClassName); + } + + } + return returnCode; + } catch (InterruptedException e) { + try { + Class processClass = process.getClass(); + Method destroyForcibly = processClass.getMethod("destroyForcibly"); + destroyForcibly.setAccessible(true); + destroyForcibly.invoke(process); + } catch (NoSuchMethodException ex) { + // we don't have destroyForcibly + process.destroy(); + } + throw new InterruptedException("Interrupted while waiting for external process."); + } + } + + /** + * Tries to get the java executable command with which the current JVM was started. + * Returns null, if the command could not be found. + * + * @return The java executable command. + */ + public static String getJavaCommandPath() { + + try { + ProcessBuilder bld = new ProcessBuilder("java", "-version"); + Process process = bld.start(); + if (process.waitFor() == 0) { + return "java"; + } + } + catch (Throwable t) { + // ignore and try the second path + } + + try { + ProcessBuilder bld = new ProcessBuilder("java.exe", "-version"); + Process process = bld.start(); + if (process.waitFor() == 0) { + return "java.exe"; + } + } + catch (Throwable t) { + // ignore and try the second path + } + + File javaHome = new File(System.getProperty("java.home")); + + String path1 = new File(javaHome, "java").getAbsolutePath(); + String path2 = new File(new File(javaHome, "bin"), "java").getAbsolutePath(); + + try { + ProcessBuilder bld = new ProcessBuilder(path1, "-version"); + Process process = bld.start(); + if (process.waitFor() == 0) { + return path1; + } + } + catch (Throwable t) { + // ignore and try the second path + } + + try { + ProcessBuilder bld = new ProcessBuilder(path2, "-version"); + Process process = bld.start(); + if (process.waitFor() == 0) { + return path2; + } + } + catch (Throwable tt) { + // no luck + } + + String path3 = new File(javaHome, "java.exe").getAbsolutePath(); + String path4 = new File(new File(javaHome, "bin"), "java.exe").getAbsolutePath(); + + try { + ProcessBuilder bld = new ProcessBuilder(path3, "-version"); + Process process = bld.start(); + if (process.waitFor() == 0) { + return path3; + } + } + catch (Throwable t) { + // ignore and try the second path + } + + try { + ProcessBuilder bld = new ProcessBuilder(path4, "-version"); + Process process = bld.start(); + if (process.waitFor() == 0) { + return path4; + } + } + catch (Throwable tt) { + // no luck + } + return null; + } + + /** + * Gets the classpath with which the current JVM was started. + * + * @return The classpath with which the current JVM was started. + */ + public static String getCurrentClasspath() { + RuntimeMXBean bean = ManagementFactory.getRuntimeMXBean(); + return bean.getClassPath(); + } + + /** + * Utility class to read the output of a process stream and forward it into a StringWriter. + */ + public static class PipeForwarder extends Thread { + + private final StringWriter target; + private final InputStream source; + + public PipeForwarder(InputStream source, StringWriter target) { + super("Pipe Forwarder"); + setDaemon(true); + + this.source = source; + this.target = target; + + start(); + } + + @Override + public void run() { + try { + int next; + while ((next = source.read()) != -1) { + target.write(next); + } + } + catch (IOException e) { + // terminate + } + } + } +} diff --git a/flink-core/src/main/java/org/apache/flink/util/HDFSCopyFromLocal.java b/flink-core/src/main/java/org/apache/flink/util/HDFSCopyFromLocal.java new file mode 100644 index 0000000000000..cf6780b01c1dd --- /dev/null +++ b/flink-core/src/main/java/org/apache/flink/util/HDFSCopyFromLocal.java @@ -0,0 +1,48 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.flink.util; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; + +import java.io.File; +import java.net.URI; + +/** + * Utility for copying from local file system to a HDFS {@link FileSystem} in an external process. + * This is required since {@code FileSystem.copyFromLocalFile} does not like being interrupted. + */ +public class HDFSCopyFromLocal { + public static void main(String[] args) throws Exception { + String localBackupPath = args[0]; + String backupUri = args[1]; + + FileSystem fs = FileSystem.get(new URI(backupUri), new Configuration()); + + fs.copyFromLocalFile(new Path(localBackupPath), new Path(backupUri)); + } + + public static void copyFromLocal(File localPath, URI remotePath) throws Exception { + ExternalProcessRunner processRunner = new ExternalProcessRunner(HDFSCopyFromLocal.class.getName(), + new String[]{localPath.getAbsolutePath(), remotePath.toString()}); + if (processRunner.run() != 0) { + throw new RuntimeException("Error while copying to remote FileSystem: " + processRunner.getErrorOutput()); + } + } +} diff --git a/flink-core/src/main/java/org/apache/flink/util/HDFSCopyToLocal.java b/flink-core/src/main/java/org/apache/flink/util/HDFSCopyToLocal.java new file mode 100644 index 0000000000000..813f768404306 --- /dev/null +++ b/flink-core/src/main/java/org/apache/flink/util/HDFSCopyToLocal.java @@ -0,0 +1,49 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.flink.util; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; + +import java.io.File; +import java.net.URI; + +/** + * Utility for copying from a HDFS {@link FileSystem} to the local file system in an external + * process. This is required since {@code FileSystem.copyToLocalFile} does not like being + * interrupted. + */ +public class HDFSCopyToLocal { + public static void main(String[] args) throws Exception { + String backupUri = args[0]; + String dbPath = args[1]; + + FileSystem fs = FileSystem.get(new URI(backupUri), new Configuration()); + + fs.copyToLocalFile(new Path(backupUri), new Path(dbPath)); + } + + public static void copyToLocal(URI remotePath, File localPath) throws Exception { + ExternalProcessRunner processRunner = new ExternalProcessRunner(HDFSCopyToLocal.class.getName(), + new String[]{remotePath.toString(), localPath.getAbsolutePath()}); + if (processRunner.run() != 0) { + throw new RuntimeException("Error while copying from remote FileSystem: " + processRunner.getErrorOutput()); + } + } +} diff --git a/flink-core/src/test/java/org/apache/flink/util/ExternalProcessRunnerTest.java b/flink-core/src/test/java/org/apache/flink/util/ExternalProcessRunnerTest.java new file mode 100644 index 0000000000000..5ebe772e42e0a --- /dev/null +++ b/flink-core/src/test/java/org/apache/flink/util/ExternalProcessRunnerTest.java @@ -0,0 +1,98 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.flink.util; + +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +public class ExternalProcessRunnerTest { + + @Test(expected = ClassNotFoundException.class) + public void testClassNotFound() throws Exception { + ExternalProcessRunner runner = new ExternalProcessRunner("MyClassThatDoesNotExist", new String[]{}); + runner.run(); + } + + @Test + public void testInterrupting() throws Exception { + + final ExternalProcessRunner runner = new ExternalProcessRunner(InfiniteLoop.class.getName(), new String[]{}); + + Thread thread = new Thread() { + @Override + public void run() { + try { + runner.run(); + } catch (InterruptedException e) { + // this is expected + } catch (Exception e) { + fail("Other exception received " + e); + } + } + }; + + thread.start(); + thread.interrupt(); + thread.join(); + } + + @Test + public void testPrintToErr() throws Exception { + final ExternalProcessRunner runner = new ExternalProcessRunner(PrintToError.class.getName(), new String[]{"hello42"}); + + int result = runner.run(); + + assertEquals(0, result); + assertEquals(runner.getErrorOutput().toString(), "Hello process hello42\n"); + } + + @Test + public void testFailing() throws Exception { + final ExternalProcessRunner runner = new ExternalProcessRunner(Failing.class.getName(), new String[]{}); + + int result = runner.run(); + + assertEquals(1, result); + // this needs to be adapted if the test changes because it contains the line number + assertEquals(runner.getErrorOutput().toString(), "Exception in thread \"main\" java.lang.RuntimeException: HEHE, I'm failing.\n" + + "\tat org.apache.flink.util.ExternalProcessRunnerTest$Failing.main(ExternalProcessRunnerTest.java:94)\n"); + } + + + public static class InfiniteLoop { + public static void main(String[] args) { + while (true) { + } + } + } + + public static class PrintToError { + public static void main(String[] args) { + System.err.println("Hello process " + args[0]); + } + } + + public static class Failing { + public static void main(String[] args) { + throw new RuntimeException("HEHE, I'm failing."); + } + } + +} diff --git a/flink-tests/pom.xml b/flink-tests/pom.xml index fe34aea157018..24b4ce9e02329 100644 --- a/flink-tests/pom.xml +++ b/flink-tests/pom.xml @@ -169,6 +169,13 @@ under the License. ${guava.version} test + + + org.apache.flink + flink-statebackend-rocksdb_2.10 + ${project.version} + test + diff --git a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/EventTimeWindowCheckpointingITCase.java b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/EventTimeWindowCheckpointingITCase.java index 5886982d17caa..9bc0040657186 100644 --- a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/EventTimeWindowCheckpointingITCase.java +++ b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/EventTimeWindowCheckpointingITCase.java @@ -25,6 +25,7 @@ import org.apache.flink.api.java.tuple.Tuple4; import org.apache.flink.configuration.ConfigConstants; import org.apache.flink.configuration.Configuration; +import org.apache.flink.contrib.streaming.state.RocksDBStateBackend; import org.apache.flink.runtime.state.AbstractStateBackend; import org.apache.flink.runtime.state.filesystem.FsStateBackend; import org.apache.flink.runtime.state.memory.MemoryStateBackend; @@ -112,6 +113,12 @@ public void initStateBackend() throws IOException { String backups = tempFolder.newFolder().getAbsolutePath(); this.stateBackend = new FsStateBackend("file://" + backups); break; + case ROCKSDB: + String rocksDb = tempFolder.newFolder().getAbsolutePath(); + String rocksDbBackups = tempFolder.newFolder().getAbsolutePath(); + + this.stateBackend = new RocksDBStateBackend(rocksDb, "file://" + rocksDbBackups, new MemoryStateBackend()); + break; } } @@ -739,6 +746,8 @@ public static Collection parameters(){ return Arrays.asList(new Object[][] { {StateBackendEnum.MEM}, {StateBackendEnum.FILE}, +// {StateBackendEnum.DB}, + {StateBackendEnum.ROCKSDB} } ); }